Mail Archives: djgpp/2003/01/07/04:22:40
Dear Sir,
I got the following errors(The file is at the bottom):
LSTM.cpp: In member function `virtual char TLSTM::LoadPar()':
LSTM.cpp:338: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[17])'
LSTM.cpp:343: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[17])'
LSTM.cpp:346: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[16])'
LSTM.cpp:347: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[21])'
LSTM.cpp:348: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[14])'
LSTM.cpp:353: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[33])'
LSTM.cpp:355: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[20])'
LSTM.cpp:356: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:358: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:360: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:364: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[15])'
LSTM.cpp:366: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[16])'
LSTM.cpp:368: no matching function for call to `std::basic_fstream<char,
std::char_traits<char> >::scan(const char[16])'
LSTM.cpp: In member function `char TLSTM::WriteWeightFile(char*)':
LSTM.cpp:2019: using typedef-name `std::iostream' after `class'
LSTM.cpp: In member function `char TLSTM::LoadWeightFile(char*)':
LSTM.cpp:2025: using typedef-name `std::iostream' after `class'
LSTM.cpp: In member function `char TLSTM::DumpAll(char*)':
LSTM.cpp:2033: using typedef-name `std::iostream' after `class'
LSTM.cpp: In member function `void TLSTM::DisplayWeights()':
LSTM.cpp:2388: using typedef-name `std::iostream' after `class'
How to fix the problem about "no matching function..." and "using typedef-name `std::iostream' after `class' "? Thanks a lot for your help.
Regards,
Y Chen
// LSTM.cpp
#include <stdlib.h>
#include <stdio.h>
#include <iostream.h>
#include <iomanip.h>
#include <new.h> // for memory error handling
//#include <ctype.h>
#include <unistd.h> // sleep
//#include <termios.h> // struct termios
#include <fcntl.h>
#include <sys/param.h>
#include <math.h>
#include <errno.h>
#include <string.h>
#include <time.h>
#include <float.h> // To have max values for imposible inits.
#include "LSTM.h"
#include "Error.h"
//// TLSTM
void TLSTM::NewTs_d_Gate(Ts_d_Gate *&sd, unsigned int size) {
sd = new Ts_d_Gate[size];
for(int i=0;i<size;i++) {
sd[i].s_d = 0;
sd[i].alpha = AlphaBase;
sd[i].h=0;
}
}
void TLSTM::NewTs_d_Gate(Ts_d_Gate &sd) {
Ts_d_Gate *psdTmp; NewTs_d_Gate(psdTmp,1);
sd.s_d = psdTmp[0].s_d;
sd.alpha = psdTmp[0].alpha;
sd.h = psdTmp[0].h;
delete psdTmp;
}
char TLSTM::Run() {
if(Init()) return 1;
for(Tri=0;Tri<NbTrials;Tri++) { // Tri loop.
if(Tri>0) if(LoadPar()) return 1;
InitNet();
//WriteWeightFile(WEIGHTLOGFILE); //exit(0);
//LoadWeightFile(WEIGHTLOGFILE);
//sprintf(cBuf, "w.init.%d",Tri); LoadWeightFile(cBuf);
//LoadWeightFile(cBuf);
//WriteWeightFile(cBuf);
//LoadWeightFile("w.init");
//if(LoadWeightFile("W")) exit(1); //ddd
//LoadWeightFile("w.hand");
//WriteWeightFile(WEIGHTLOGFILE);
// Run and Train the net.
//Test(); exit(0); //ddd
//GetOnlineProblemStatistics(); exit(0); //ddd
do { // Epo loop.
Epo++;
#ifdef DO_ONLINE_PAT_PRODUCTION
// This Seq will be overwritten, we init the Nb variables here.
ONLINE_PAT_FUNCNAME(sTrainData, true);
#else
if(Generate_Pat && Generate_Pat_Each_Epoch>0) {
if((unsigned int)fmod(Epo, Generate_Pat_Each_Epoch) == 0)
GeneratePattern(sTrainData, Pat_FuncName);
}
if(MixTrainSeqs) MixTrainSequences();
#endif
ClassesWrong=0; MSEEpo=0; PatCorrect=0; PatCount=0;
SeqOnline=0; SeqOnlinePatCount=0;
// Seq loop.
for(Seq=0;Seq<sTrainData.NbSeq;FreezeSeqLoop ? Seq=0 : Seq++) {
MSESeq=0; PatWrong=0;
#ifndef DO_ONLINE_PAT_PRODUCTION
ResetNet();
#else
if(!SeqOnline) ResetNet(); //For sequential online
#endif
#ifdef DO_ONLINE_PAT_PRODUCTION
#if defined(NO_RESET_AFTER_ONLINE_SEQ) || defined(SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE)
if(SeqOnline) ONLINE_PAT_FUNCNAME(sTrainData, false);
else ONLINE_PAT_FUNCNAME(sTrainData, true);
#else
ONLINE_PAT_FUNCNAME(sTrainData, true);
#endif
#endif
// if(PatCorrectTest>10)
// {cout<< "\n";cout.flush();}//Debug on-line.
//DumpAll("dump.init");
//DisplayNet(sTrainData); DisplayWeights(); KeyCheck();
//AlphaPredict = Alpha / sTrainData.NbPat[Seq]; // AlphaPredict
for(Pat=0;Pat<sTrainData.NbPat[Seq];Pat++) { // Pat loop.
// if(PatCorrectTest>10) { // Debug on-line pat generation. // ddd
// cout << Epo << "-" << Seq << "-" << SeqOnline
// << "-" << Pat << " :";
// cout << "[" << sTrainData.NbPat[0]<< "] ";
// for(int Val=0;Val<sTrainData.NbVal;Val++)
// cout << ((sTrainData.SeqList[Seq])[Pat])[Val] << " ";
// cout << " = " << sTrainData.NbClass[Seq];
// cout << " -> (" << MSEPat << ") " << PatCorrect;
// cout << "\n"; cout.flush();
// }
ForwardPass(sTrainData,FreezeEndBlock,NbMemoBlocks);
//WriteOutLogFile(OUT_LOGFILE);
#ifdef USE_LOCAL_ALPHA_SEQUENCEWISE
BackwardPass(FreezeEndBlock,NbMemoBlocks);
#endif
if((Pat==sTrainData.NbPat[Seq]-1) || SetStepTarget ||
(NbPredictNextIn>0) || (NbPredictClass>0)) {
#ifdef USE_ALPHA_DECAY_IN_SEQ
if(SetStepTarget) {
#ifdef ALPHA_DECAY_IN_SEQ_LINEAR
AlphaDecayInSeq = USE_ALPHA_DECAY_IN_SEQ/
(Pat+USE_ALPHA_DECAY_IN_SEQ);
#elif
if(Pat*USE_ALPHA_DECAY_IN_SEQ>709) AlphaDecayInSeq=0;
else AlphaDecayInSeq = EXP(-(Pat*USE_ALPHA_DECAY_IN_SEQ));
#endif
} else {
#ifdef ALPHA_DECAY_IN_SEQ_LINEAR
AlphaDecayInSeq = USE_ALPHA_DECAY_IN_SEQ/
(SeqOnline+USE_ALPHA_DECAY_IN_SEQ);
#elif
if(SeqOnline*USE_ALPHA_DECAY_IN_SEQ>709) AlphaDecayInSeq=0;
else AlphaDecayInSeq =
EXP(-(SeqOnline*USE_ALPHA_DECAY_IN_SEQ));
#endif
}
// cout << Pat << "-" << Seq << "-" << Epo << ":"
// << AlphaDecayInSeq << " ";
#endif
#ifndef USE_LOCAL_ALPHA_SEQUENCEWISE
BackwardPass(FreezeEndBlock,NbMemoBlocks);
#endif
PatStatistics(sTrainData);
#ifdef DO_ONLINE_PAT_PRODUCTION
if(SetStepTarget) {
if(!PatWrong)
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
ONLINE_PAT_FUNCNAME(sTrainData, false);
#else
; // Generate seq only before Pat loop.
#endif
else break; // PatWrong.
//if(ReberGrammarState==-1) ResetNet();
}
#endif
}
// if(Epo==7384) {
// sprintf(cBuf, "w.end%d.%d",Epo,Pat);WriteWeightFile(cBuf);
// if(Pat>840) {
// sprintf(cBuf, "dump.%d.%d",Epo,Pat);DumpAll(cBuf); }
// }
//sprintf(cBuf, "w.Pat.%d",Pat); WriteWeightFile(cBuf); exit(0);
//sprintf(cBuf, "dump.Pat.%d",Pat);DumpAll(cBuf);
//if(Pat>=3) exit(0);
//DisplayNet(sTrainData); DisplayWeights(); KeyCheck();
//if(PatCorrectTest>90) Dump_Inner_States(); //ddd
//printf(cBuf, "dump.%d.%d",Epo,Pat);DumpAll(cBuf);
//if(Epo>1000) exit(0);
//Dump_Inner_States();
} // End Pat loop.
#ifdef UPDATE_WEIGHTS_AFTER_SEQ
ExecuteWeightChanges(FreezeEndBlock,NbMemoBlocks);
#endif
SeqStatistics(sTrainData);
//sprintf(cBuf, "w.Seq.%d",Seq); WriteWeightFile(cBuf); exit(0);
//DisplayNet(sTrainData); DisplayWeights(); KeyCheck();
#ifdef DO_ONLINE_PAT_PRODUCTION
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
if(!SetStepTarget)
#endif
if(!PatWrong) {
SeqOnline++;
if(PatCorrect>=MaxOnlineStreamLengthTrain) break;
} else break;
#endif
} // End Seq loop.
//DumpSomeAlphas();
EpoStatistics(sTrainData);
//ExecuteWeightChanges(FreezeEndBlock,NbMemoBlocks);
//sprintf(cBuf, "w.Epo.%d",Epo); WriteWeightFile(cBuf);
//sprintf(cBuf, "%s.%d", OUT_LOGFILE,Epo); rename(OUT_LOGFILE,cBuf);
//if(Epo==7384) DumpAll("dump.end.log");
//if((unsigned int)fmod(Epo,1000) == 0) {
//sprintf(cBuf, "w.end%d.%d",Tri,Epo);WriteWeightFile(cBuf); }
//WriteWeightFile("w");)
if(StopLern) EpochsAfterLearned--;
#ifdef DO_ONLINE_PAT_PRODUCTION
if((Epo>=MaxEpochs)||(PatCorrect>=MaxOnlineStreamLengthTrain)
||(MSEEpo<MSEStop)) StopLern=true;
// StopLearn when the taing set is leerand -> test.
if(StopLern && (Epo<MaxEpochs)) {
if(LastTestEpo+TestMaxEach_Epo<Epo) // Not test too much.
Test(); // If not just tested during the EpoStatistics.
// Here StopLern only then trainig requirements met..
//StopLern=((PatCorrectTest>=MaxOnlineStreamLengthTest) ||
// (TestMSEEpo<MSEStop));
}
// StopLearn also when the test is learnd but the trainig not.
// The first or is for the case that we give up (MaxEpochs).
// Test results may come from test during EpoStatistics.
StopLern=(Epo>=MaxEpochs) ||
((PatCorrectTest>=MaxOnlineStreamLengthTest) ||
(TestMSEEpo<MSEStop));
#else
if(!ClassesWrong||(Epo>=MaxEpochs)||(MSEEpo<MSEStop)) StopLern=true;
// Test when Traing set learned.
//if(LastTestEpo+TestMaxEach_Epo<Epo) { // Not test too much
//if(StopLern&&(Epo<MaxEpochs)) StopLern = !Test();
//} else StopLern = false;
#endif
} while (!StopLern || (EpochsAfterLearned>0));// End Epo loop.
Test();// Final Test
//if(Epo>=MaxEpochs) { Test();}// Final test if unsolved.
//if(Epo>=MaxEpochs) {OutputDebug=true; Test();}// ddd
//if(Epo<MaxEpochs){OutputDebug=true; Test();}//ddd
OutputDebug=false; //ddd
#ifdef BUFFER_EPO_LOG
// Flush the buffers to files.
if(TrainMSEEpoLogBuf.NbVal)
FlushEpoLogBuf(MSELOGFILE, TrainMSEEpoLogBuf);
if(TrainErrEpoLogBuf.NbVal)
FlushEpoLogBuf(ERRORLOGFILE, TrainErrEpoLogBuf);
if(TestErrEpoLogBuf.NbVal)
FlushEpoLogBuf(TEST_ERRORLOGFILE, TestErrEpoLogBuf);
if(TestMSEEpoLogBuf.NbVal)
FlushEpoLogBuf(TEST_MSELOGFILE, TestMSEEpoLogBuf);
#endif
sprintf(cBuf, "w.end%d.%d",Tri,Epo); WriteWeightFile(cBuf);
if(NbTrials>1) {
sprintf(cBuf, "%s.%d", ERRORLOGFILE,Tri); rename(ERRORLOGFILE,cBuf);
sprintf(cBuf, "%s.%d", MSELOGFILE,Tri); rename(MSELOGFILE,cBuf);
if(DirContainsFile(true, TEST_ERRORLOGFILE)) {
sprintf(cBuf, "%s.%d", TEST_ERRORLOGFILE,Tri);
rename(TEST_ERRORLOGFILE,cBuf); }
if(DirContainsFile(true, TEST_MSELOGFILE)) {
sprintf(cBuf, "%s.%d", TEST_MSELOGFILE,Tri);
rename(TEST_MSELOGFILE,cBuf); }
if(GrowNet) {
sprintf(cBuf, "%s.%d", GROWLOGFILE,Tri); rename(GROWLOGFILE,cBuf); }
if(DirContainsFile(true, WEIGHTMEAN_LOGFILE)) {
sprintf(cBuf, "%s.%d", WEIGHTMEAN_LOGFILE,Tri);
rename(WEIGHTMEAN_LOGFILE,cBuf); }
if(DirContainsFile(true, ALPHAMEAN_LOGFILE)) {
sprintf(cBuf, "%s.%d", ALPHAMEAN_LOGFILE,Tri);
rename(ALPHAMEAN_LOGFILE,cBuf); }
if(DirContainsFile(true, INNERSTATE_DUMP_LOGFILE)) {
sprintf(cBuf, "%s.%d", INNERSTATE_DUMP_LOGFILE,Tri);
rename(INNERSTATE_DUMP_LOGFILE,cBuf); }
if(DirContainsFile(true, "AlphaDump.log")) {
sprintf(cBuf, "%s.%d", "AlphaDump.log",Tri);
rename("AlphaDump.log",cBuf); }
}
DeleteNet();
} // End Tri loop.
cout << "LSTM Done." << endl;
}
void TLSTM::InitNet() {
InitMemoBlocks(0,NbMemoBlocks);
// Init other units.
// Init Out units.
Out = new TOut[NbOut];
for(int iO=0;iO<NbOut;iO++) {
NewWeight(Out[iO].w_Bias,OutUnitBiasInitWeight);
#ifndef NO_IN_OUT_SHORTCUTS
NewWeight(Out[iO].w_In,NbIn);
#endif
NewWeight(Out[iO].w_Hidden,NbHidden);
Out[iO].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
for(int iB=0;iB<NbMemoBlocks;iB++)
NewWeight(Out[iO].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
}
// Init Hidden units.
Hidden = new THidden[NbHidden];
for(int iH=0;iH<NbHidden;iH++) {
NewWeight(Hidden[iH].w_Bias,HiddenUnitBiasInitWeight);
NewWeight(Hidden[iH].w_In, NbIn);
Hidden[iH].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
for(int iB=0;iB<NbMemoBlocks;iB++)
NewWeight(Hidden[iH].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
}
// Init NextIn predict.
// Init PredictNextIn units.
PredictNextIn = new TPredictNextIn[NbPredictNextIn];
for(int iP=0;iP<NbPredictNextIn;iP++) {
NewWeight(PredictNextIn[iP].w_In, NbIn);
NewWeight(PredictNextIn[iP].w_InClass, NbOut);
PredictNextIn[iP].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
for(int iB=0;iB<NbMemoBlocks;iB++)
NewWeight(PredictNextIn[iP].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
}
// Init NextIn units.
if(NbPredictNextIn>0) {
NextIn = new TNextIn[NbIn];
for(int iO=0;iO<NbIn;iO++)
NewWeight(NextIn[iO].w_PredictNextIn, NbPredictNextIn);
}
// Init Class predict.
// Init PredictClass units.
PredictClass = new TPredictClass[NbPredictClass];
for(int iP=0;iP<NbPredictClass;iP++) {
PredictClass[iP].w_Cell = new (TWeight *)[MAX_MEMOBLOCKS];
for(int iB=0;iB<NbMemoBlocks;iB++)
NewWeight(PredictClass[iP].w_Cell[iB], MemoBlock[iB].MemoBlockSize);
}
// Init PredictClassOut units.
if(NbPredictClass>0) {
PredictClassOut = new TPredictClassOut[NbOut];
for(int iO=0;iO<NbOut;iO++)
NewWeight(PredictClassOut[iO].w_PredictClass, NbPredictClass);
}
// Init statistics and other working variables
// (only what is not done in RUN).
PatCorrectTest=0; TestMSEEpo=DBL_MAX; // First test result for StopLern.
LastTestEpo=0;
FreezeEndBlock=0; // Worm start.
Epo=0;
LogErrRecMean=0; PartLogErrRecMean=0; NbLogedErr=0;
WeightMean=0; AlphaMean=0; AlphaStd=0; //WPC=0;
StopLern=false; MaxPatCorrectTrain=0;
#ifdef BUFFER_EPO_LOG
TrainMSEEpoLogBuf.Reset(); TrainErrEpoLogBuf.Reset();
TestErrEpoLogBuf.Reset(); TestMSEEpoLogBuf.Reset();
#endif
//Alpha = AlphaBase + AlphaError;
NbPredictOut =0;
if(NbPredictNextIn>0) NbPredictOut +=NbIn;
if(NbPredictClass>0) NbPredictOut +=NbOut;
SetTopoStatistics();
}
char TLSTM::LoadPar() {
double NotUsed;
if (TNeuralNetBase::LoadPar()) return 1;
if (OpenFile(cParFilename, ios::in)) return 1;
if (ReadComment()) return 1;
// Scan for the starting point in the parameter file.
pFile->scan("NbPredictNextIn:");
while(!pFile->good()) {
pFile->seekg(1, pFile->cur);
if(pFile->eof()) { cout << "eof\n"; return 1;}
pFile->clear();
pFile->scan("NbPredictNextIn:"); // First parameter of LSTM.
}
*pFile >> NbPredictNextIn;
pFile->scan("NbPredictClass:"); *pFile >> NbPredictClass;
pFile->scan("InternalBlockConect:"); *pFile >> InternalBlockConect;
pFile->scan("NbMemoBlocks:"); *pFile >> NbMemoBlocks;
if(NbMemoBlocks>MAX_MEMOBLOCKS) {
cerr << "NbMemoBlocks>MAX_MEMOBLOCKS\n"; exit(1); }
MemoBlock = new (TMemoBlock)[NbMemoBlocks];
//MemoBlock = new (TMemoBlock)[MAX_MEMOBLOCKS];
pFile->scan("# parameters for each memo block");
for(int iB=0;iB<NbMemoBlocks;iB++) {
pFile->scan("# memo block number"); pFile->ignore(1000, '\n');
pFile->scan("MemoBlockSize:");
*pFile >> MemoBlock[iB].MemoBlockSize;
pFile->scan("BegSrcBlockNb:");
*pFile >> MemoBlock[iB].BegSrcBlockNb;
pFile->scan("EndSrcBlockNb:");
*pFile >> MemoBlock[iB].EndSrcBlockNb;
if(MemoBlock[iB].EndSrcBlockNb==-1)
MemoBlock[iB].EndSrcBlockNb = NbMemoBlocks;
pFile->scan("InputGateBias:");
*pFile >> MemoBlock[iB].InGate.w_Bias.w;
pFile->scan("OutputGateBias:");
*pFile >> MemoBlock[iB].OutGate.w_Bias.w;
pFile->scan("ForgetGateBias:");
#ifdef USE_FORGET_GATES
*pFile >> MemoBlock[iB].FgGate.w_Bias.w;
#else
*pFile >> NotUsed;
#endif
}
return CloseFile();
}
void TLSTM::InitMemoBlocks(unsigned int BegBlock, unsigned int EndBlock) {
// MemoBlocks (newed in LoadPar() and partial inited).
// For easier indexing all arrays are newed from zero instead from
// MemoBlock[iB].BegSrcBlockNb. For the **variables, never
// used *var[] indices, are not newed (no difference in indexing).
// BegSrcBlockNb will be zero(or small) in most cases anyway.
// But instead of newing until NbMemoBlocks we will use
// MemoBlock[iB].EndSrcBlockNb (not any more for full Add).
for(int iB=BegBlock;iB<EndBlock;iB++) {
// InGate.
NewWeight(MemoBlock[iB].InGate.w_Bias,MemoBlock[iB].InGate.w_Bias.w);
NewWeight(MemoBlock[iB].InGate.w_In, NbIn);
#ifdef CONNECT_GATES_TO_S
NewWeight(MemoBlock[iB].InGate.w_s, MemoBlock[iB].MemoBlockSize);
#endif
MemoBlock[iB].InGate.w_Cell =
new (TWeight *)[MAX_MEMOBLOCKS];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
NewWeight(MemoBlock[iB].InGate.w_Cell[iiB],
MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
NewWeight(MemoBlock[iB].InGate.w_InGate, MAX_MEMOBLOCKS);
NewWeight(MemoBlock[iB].InGate.w_OutGate ,MAX_MEMOBLOCKS);
#endif
// OutGate.
NewWeight(MemoBlock[iB].OutGate.w_Bias,MemoBlock[iB].OutGate.w_Bias.w);
NewWeight(MemoBlock[iB].OutGate.w_In ,NbIn);
#ifdef CONNECT_GATES_TO_S
NewWeight(MemoBlock[iB].OutGate.w_s, MemoBlock[iB].MemoBlockSize);
#endif
MemoBlock[iB].OutGate.w_Cell =
new (TWeight *)[MAX_MEMOBLOCKS]; //[MemoBlock[iB].EndSrcBlockNb];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
NewWeight(MemoBlock[iB].OutGate.w_Cell[iiB],
MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
NewWeight(MemoBlock[iB].OutGate.w_InGate ,MAX_MEMOBLOCKS);
NewWeight(MemoBlock[iB].OutGate.w_OutGate ,MAX_MEMOBLOCKS);
#endif
#ifdef USE_FORGET_GATES
// FgGate.
NewWeight(MemoBlock[iB].FgGate.w_Bias,MemoBlock[iB].FgGate.w_Bias.w);
NewWeight(MemoBlock[iB].FgGate.w_In, NbIn);
#ifdef CONNECT_GATES_TO_S
NewWeight(MemoBlock[iB].FgGate.w_s, MemoBlock[iB].MemoBlockSize);
#endif
MemoBlock[iB].FgGate.w_Cell =
new (TWeight *)[MAX_MEMOBLOCKS];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
NewWeight(MemoBlock[iB].FgGate.w_Cell[iiB],
MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
NewWeight(MemoBlock[iB].FgGate.w_InGate, MAX_MEMOBLOCKS);
NewWeight(MemoBlock[iB].FgGate.w_OutGate ,MAX_MEMOBLOCKS);
#endif
#endif
// Cells.
MemoBlock[iB].Cell = new (TCell)[MemoBlock[iB].MemoBlockSize];
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef USE_CELL_BIAS
NewWeight(MemoBlock[iB].Cell[iC].w_Bias,0); // Random init.
#endif
NewWeight(MemoBlock[iB].Cell[iC].w_In ,NbIn);
MemoBlock[iB].Cell[iC].w_Cell =
new (TWeight *)[MAX_MEMOBLOCKS]; //[MemoBlock[iB].EndSrcBlockNb];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
NewWeight(MemoBlock[iB].Cell[iC].w_Cell[iiB],
MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
NewWeight(MemoBlock[iB].Cell[iC].w_InGate, MAX_MEMOBLOCKS);
NewWeight(MemoBlock[iB].Cell[iC].w_OutGate, MAX_MEMOBLOCKS);
#endif
}
}
// Init the derivatives s_d.
for(int iB=BegBlock;iB<EndBlock;iB++) {
// For Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
MemoBlock[iB].Cell[iC].s_d_In = new double[NbIn];
MemoBlock[iB].Cell[iC].s_d_Cell = new (double *)[MAX_MEMOBLOCKS];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
MemoBlock[iB].Cell[iC].s_d_Cell[iiB] =
new double[MemoBlock[iiB].MemoBlockSize];
#ifdef CONNECT_TO_GATES
MemoBlock[iB].Cell[iC].s_d_InGate =
new double[MAX_MEMOBLOCKS];
MemoBlock[iB].Cell[iC].s_d_OutGate =
new double[MAX_MEMOBLOCKS];
#endif
// For InGate.
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_Bias);
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_In,NbIn);
#ifdef CONNECT_GATES_TO_S
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_s,
MemoBlock[iB].MemoBlockSize);
#endif
MemoBlock[iB].Cell[iC].s_d_InGate_Cell =
new (Ts_d_Gate *)[MAX_MEMOBLOCKS];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB],
MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_InGate,
MAX_MEMOBLOCKS);
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_OutGate,
MAX_MEMOBLOCKS);
#endif
#ifdef USE_FORGET_GATES
// For FgGate.
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_Bias);
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_In,NbIn);
#ifdef CONNECT_GATES_TO_S
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_s,
MemoBlock[iB].MemoBlockSize);
#endif
MemoBlock[iB].Cell[iC].s_d_FgGate_Cell =
new (Ts_d_Gate *)[MAX_MEMOBLOCKS];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB],
MemoBlock[iiB].MemoBlockSize);
#ifdef CONNECT_TO_GATES
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_InGate,
MAX_MEMOBLOCKS);
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate,
MAX_MEMOBLOCKS);
#endif
#endif
}
}
}
void TLSTM::AddMemoBlockToExistingBlocks(unsigned int BegBlock,
unsigned int EndBlock) {
unsigned int Ml = NbMemoBlocks-1; // last Block shorthand.
for(int iB=BegBlock;iB<EndBlock;iB++) {
// Always fully connected.
MemoBlock[iB].EndSrcBlockNb = NbMemoBlocks;
// InGate.
NewWeight(MemoBlock[iB].InGate.w_Cell[Ml], MemoBlock[Ml].MemoBlockSize);
// OutGate.
NewWeight(MemoBlock[iB].OutGate.w_Cell[Ml], MemoBlock[Ml].MemoBlockSize);
#ifdef USE_FORGET_GATES
// FgGate.
NewWeight(MemoBlock[iB].FgGate.w_Cell[Ml], MemoBlock[Ml].MemoBlockSize);
#endif
// Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
NewWeight(MemoBlock[iB].Cell[iC].w_Cell[Ml],
MemoBlock[Ml].MemoBlockSize);
}
// Init the derivatives s_d.
for(int iB=BegBlock;iB<EndBlock;iB++) {
// For Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
MemoBlock[iB].Cell[iC].s_d_Cell[Ml] =
new double[MemoBlock[Ml].MemoBlockSize];
// For InGate.
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_InGate_Cell[Ml],
MemoBlock[Ml].MemoBlockSize);
#ifdef USE_FORGET_GATES
// For FgGate.
NewTs_d_Gate(MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[Ml],
MemoBlock[Ml].MemoBlockSize);
#endif
}
}
// Set Init Weights different than in NewWeight().
// Set MemoBlock weights.
for(int iB=BegBlock;iB<EndBlock;iB++) {
// InGate weights. The bias was already set in LoadPar.
#ifdef CONNECT_TO_GATES
MemoBlock[iB].InGate.w_InGate[Ml].w = 0;
MemoBlock[iB].InGate.w_OutGate[Ml].w = 0;
#endif
for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
MemoBlock[iB].InGate.w_Cell[Ml][iiC].w = 0;
// OutGate weights.
#ifdef CONNECT_TO_GATES
MemoBlock[iB].OutGate.w_InGate[Ml].w = 0;
MemoBlock[iB].OutGate.w_OutGate[Ml].w = 0;
#endif
for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
MemoBlock[iB].OutGate.w_Cell[Ml][iiC].w= 0;
#ifdef USE_FORGET_GATES
// FgGate weights. The bias was already set in LoadPar.
#ifdef CONNECT_TO_GATES
MemoBlock[iB].FgGate.w_InGate[Ml].w = 0;
MemoBlock[iB].FgGate.w_OutGate[Ml].w = 0;
#endif
for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
MemoBlock[iB].FgGate.w_Cell[Ml][iiC].w = 0;
#endif
// Cells weights.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef CONNECT_TO_GATES
MemoBlock[iB].Cell[iC].w_InGate[Ml].w =0;
MemoBlock[iB].Cell[iC].w_OutGate[Ml].w =0;
#endif
for(int iiC=0;iiC<MemoBlock[Ml].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].w_Cell[Ml][iiC].w = 0;
}
}
}
void TLSTM::DeleteNet() {
// Delete MemoBlocks.
for(int iB=0;iB<NbMemoBlocks;iB++) {
// InGate.
delete MemoBlock[iB].InGate.w_In;
#ifdef CONNECT_GATES_TO_S
delete MemoBlock[iB].InGate.w_s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].InGate.w_Cell[iiB];
delete MemoBlock[iB].InGate.w_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].InGate.w_InGate;
delete MemoBlock[iB].InGate.w_OutGate;
#endif
// OutGate.
delete MemoBlock[iB].OutGate.w_In;
#ifdef CONNECT_GATES_TO_S
delete MemoBlock[iB].OutGate.w_s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].OutGate.w_Cell[iiB];
delete MemoBlock[iB].OutGate.w_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].OutGate.w_InGate;
delete MemoBlock[iB].OutGate.w_OutGate;
#endif
#ifdef USE_FORGET_GATES
// FgGate.
delete MemoBlock[iB].FgGate.w_In;
#ifdef CONNECT_GATES_TO_S
delete MemoBlock[iB].FgGate.w_s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].FgGate.w_Cell[iiB];
delete MemoBlock[iB].FgGate.w_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].FgGate.w_InGate;
delete MemoBlock[iB].FgGate.w_OutGate;
#endif
#endif
// Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
delete MemoBlock[iB].Cell[iC].w_In;
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].Cell[iC].w_Cell[iiB];
delete MemoBlock[iB].Cell[iC].w_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].Cell[iC].w_InGate;
delete MemoBlock[iB].Cell[iC].w_OutGate;
#endif
}
delete MemoBlock[iB].Cell;
}
// Delete the derivatives s_d.
for(int iB=0;iB<NbMemoBlocks;iB++) {
// For Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
delete MemoBlock[iB].Cell[iC].s_d_In;
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].Cell[iC].s_d_Cell[iiB];
delete MemoBlock[iB].Cell[iC].s_d_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].Cell[iC].s_d_InGate;
delete MemoBlock[iB].Cell[iC].s_d_OutGate;
#endif
// For InGate.
delete MemoBlock[iB].Cell[iC].s_d_InGate_In;
#ifdef CONNECT_GATES_TO_S
delete MemoBlock[iB].Cell[iC].s_d_InGate_s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB];
delete MemoBlock[iB].Cell[iC].s_d_InGate_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].Cell[iC].s_d_InGate_InGate;
delete MemoBlock[iB].Cell[iC].s_d_InGate_OutGate;
#endif
#ifdef USE_FORGET_GATES
// FgGate.
delete MemoBlock[iB].Cell[iC].s_d_FgGate_In;
#ifdef CONNECT_GATES_TO_S
delete MemoBlock[iB].Cell[iC].s_d_FgGate_s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++)
delete MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB];
delete MemoBlock[iB].Cell[iC].s_d_FgGate_Cell;
#ifdef CONNECT_TO_GATES
delete MemoBlock[iB].Cell[iC].s_d_FgGate_InGate;
delete MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate;
#endif
#endif
}
}
delete MemoBlock;
// Delete other units.
// Delete Out units.
for(int iO=0;iO<NbOut;iO++) {
#ifndef NO_IN_OUT_SHORTCUTS
delete Out[iO].w_In;
#endif
for(int iB=0;iB<NbMemoBlocks;iB++)
delete Out[iO].w_Cell[iB];
delete Out[iO].w_Cell;
}
delete Out;
// Delete Hidden units.
for(int iH=0;iH<NbHidden;iH++) {
delete Hidden[iH].w_In;
for(int iB=0;iB<NbMemoBlocks;iB++)
delete Hidden[iH].w_Cell[iB];
delete Hidden[iH].w_Cell;
}
delete Hidden;
// Delete PredictNextIn units.
for(int iP=0;iP<NbPredictNextIn;iP++) {
delete PredictNextIn[iP].w_In;
delete PredictNextIn[iP].w_InClass;
for(int iB=0;iB<NbMemoBlocks;iB++)
delete PredictNextIn[iP].w_Cell[iB];
delete PredictNextIn[iP].w_Cell;
}
delete PredictNextIn;
// Delete NextIn units.
if(NbPredictNextIn>0) {
for(int iO=0;iO<NbIn;iO++) delete NextIn[iO].w_PredictNextIn;
delete NextIn;
}
// Delete PredictClass units.
for(int iP=0;iP<NbPredictClass;iP++) {
for(int iB=0;iB<NbMemoBlocks;iB++)
delete PredictClass[iP].w_Cell[iB];
delete PredictClass[iP].w_Cell;
}
delete PredictClass;
// Delete PredictClassOut units.
if(NbPredictClass>0) {
for(int iO=0;iO<NbOut;iO++)
delete PredictClassOut[iO].w_PredictClass;
delete PredictClassOut;
}
}
char TLSTM::AddMemoBlock(int BegSrcBlockNb, int EndSrcBlockNb,
int MemoBlockSize) {
if(NbMemoBlocks>=MAX_MEMOBLOCKS) return 1;
// Add Block.
if(FreezeAndGrow) FreezeEndBlock=NbMemoBlocks; // Only the new block.
NbMemoBlocks++;
if(BegSrcBlockNb==-1) BegSrcBlockNb = NbMemoBlocks-1; // Only self.
if(EndSrcBlockNb==-1) EndSrcBlockNb = NbMemoBlocks;
MemoBlock[NbMemoBlocks-1].MemoBlockSize = MemoBlockSize;
MemoBlock[NbMemoBlocks-1].BegSrcBlockNb = BegSrcBlockNb;
MemoBlock[NbMemoBlocks-1].EndSrcBlockNb = EndSrcBlockNb;
MemoBlock[NbMemoBlocks-1].InGate.w_Bias.w = -1;
MemoBlock[NbMemoBlocks-1].OutGate.w_Bias.w = -1;
#ifdef USE_FORGET_GATES
MemoBlock[NbMemoBlocks-1].FgGate.w_Bias.w = +1;
#endif
InitMemoBlocks(NbMemoBlocks-1, NbMemoBlocks);
if(GrowFullyNotCascade) AddMemoBlockToExistingBlocks(0,NbMemoBlocks-1);
// Add new block the Out src.
for(int iO=0;iO<NbOut;iO++)
NewWeight(Out[iO].w_Cell[NbMemoBlocks-1],
MemoBlock[NbMemoBlocks-1].MemoBlockSize);
// Add new block the PredictNextIn src.
for(int iP=0;iP<NbPredictNextIn;iP++)
NewWeight(PredictNextIn[iP].w_Cell[NbMemoBlocks-1],
MemoBlock[NbMemoBlocks-1].MemoBlockSize);
// Add new block the PredictClass src.
for(int iP=0;iP<NbPredictClass;iP++)
NewWeight(PredictClass[iP].w_Cell[NbMemoBlocks-1],
MemoBlock[NbMemoBlocks-1].MemoBlockSize);
// Refresh statistics.
SetTopoStatistics();
return 0;
}
void TLSTM::ResetNet() {
// Reset MemoBlock varables.
for(int iB=0;iB<NbMemoBlocks;iB++) {
// InGate variables.
MemoBlock[iB].InGate.y=0.0;
// OutGate variables.
MemoBlock[iB].OutGate.y=0.0;
// FgGate variables.
#ifdef USE_FORGET_GATES
MemoBlock[iB].FgGate.y=0.0; // Even though No connection form the gates.
#endif
// Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
// Cell variables.
MemoBlock[iB].Cell[iC].s=0;
MemoBlock[iB].Cell[iC].y=0.0;
// s_d for cells.
#ifdef USE_CELL_BIAS
MemoBlock[iB].Cell[iC].s_d_Bias=0;
#endif
for(int iI=0;iI<NbIn;iI++)
MemoBlock[iB].Cell[iC].s_d_In[iI]=0;
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
MemoBlock[iB].Cell[iC].s_d_InGate[iiB]=0;
MemoBlock[iB].Cell[iC].s_d_OutGate[iiB]=0;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC]=0;
}
// s_d for InGates.
MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d=0;
for(int iI=0;iI<NbIn;iI++)
MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d=0;
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d=0;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d=0;
MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d=0;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d=0;
}
#ifdef USE_FORGET_GATES
// s_d for FgGates.
MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d=0;
for(int iI=0;iI<NbIn;iI++)
MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d=0;
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d=0;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d=0;
MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d=0;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d=0;
}
#endif
}
}
// Reset Out units varables. no.
// Reset Hidden units varables.
for(int iH=0;iH<NbHidden;iH++) Hidden[iH].y=0;
// Reset PredictNextIn units varables.
for(int iP=0;iP<NbPredictNextIn;iP++) PredictNextIn[iP].y=0;
// Reset NextIn units varables not necessary.
// Reset PredictClass units varables.
for(int iP=0;iP<NbPredictClass;iP++) PredictClass[iP].y=0;
// Reset PredictClassOut units varables not necessary.
}
void TLSTM::ForwardPass(struct TPatData &PatData, unsigned int BegBlockDeriv,
unsigned int EndBlockDeriv) {
//TopoCheck(); exit(0);
// One step process: update Hidden, Predict, MemoBlocks than Out.
// Temporal order within the MemoBlocks:
// i) net_InGate, net_FgGate, net_cell (gates see s(t-1)(if connected))
// ii) y_InGate, y_FgGate -> s(t)
// ii) net_OutGate (with s(t)(if connected)), y_OutGate -> y_cell.
// Input to the Hidden units (net).
for(int iH=0;iH<NbHidden;iH++) {
//Hidden[iH].net=0; // Reset with bias.
Hidden[iH].net = Hidden[iH].w_Bias.w;
for(int iI=0;iI<NbIn;iI++) // From Input.
Hidden[iH].net += Hidden[iH].w_In[iI].w *
((PatData.SeqList[Seq])[Pat])[iI];
for(int iB=0;iB<NbMemoBlocks;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
Hidden[iH].net += Hidden[iH].w_Cell[iB][iC].w *
MemoBlock[iB].Cell[iC].y;
}
// // Input to the PredictNextIn units (net).
// for(int iP=0;iP<NbPredictNextIn;iP++) {
// PredictNextIn[iP].net=0;
// for(int iI=0;iI<NbIn;iI++) // From Input.
// PredictNextIn[iP].net += PredictNextIn[iP].w_In[iI].w *
// ((PatData.SeqList[Seq])[Pat])[iI];
// for(int iI=0;iI<NbOut;iI++) // From constant InClass.
// PredictNextIn[iP].net += PredictNextIn[iP].w_InClass[iI].w *
// ((PatData.SeqList[Seq])[PatData.NbPat[Seq]-1])[NbIn+iI];
// for(int iB=0;iB<NbMemoBlocks;iB++)
// for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
// PredictNextIn[iP].net += PredictNextIn[iP].w_Cell[iB][iC].w *
// MemoBlock[iB].Cell[iC].y;
// }
// // Input to the NextIn units (net).
// if(NbPredictNextIn>0) {
// for(int iO=0;iO<NbIn;iO++) {
// NextIn[iO].net=0;
// for(int iP=0;iP<NbPredictNextIn;iP++)
// NextIn[iO].net +=NextIn[iO].w_PredictNextIn[iP].w *
// PredictNextIn[iP].y;
// }
// }
// // Input to the PredictClass units (net).
// for(int iP=0;iP<NbPredictClass;iP++) {
// PredictClass[iP].net=0;
// for(int iB=0;iB<NbMemoBlocks;iB++)
// for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
// PredictClass[iP].net += PredictClass[iP].w_Cell[iB][iC].w *
// MemoBlock[iB].Cell[iC].y;
// }
// // Input to the PredictClassOut units (net).
// if(NbPredictClass>0) {
// for(int iO=0;iO<NbOut;iO++) {
// PredictClassOut[iO].net=0;
// for(int iP=0;iP<NbPredictClass;iP++)
// PredictClassOut[iO].net += PredictClassOut[iO].w_PredictClass[iP].w *
// PredictClass[iP].y;
// }
// }
// Input to MemoBlocks (net).
for(int iB=0;iB<NbMemoBlocks;iB++) {
// net InGate.
//MemoBlock[iB].InGate.net=0; // Reset with Bias connection.
MemoBlock[iB].InGate.net = MemoBlock[iB].InGate.w_Bias.w;
for(int iI=0;iI<NbIn;iI++) // From Input.
MemoBlock[iB].InGate.net += MemoBlock[iB].InGate.w_In[iI].w *
((PatData.SeqList[Seq])[Pat])[iI];
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
MemoBlock[iB].InGate.net += MemoBlock[iB].InGate.w_s[iiC].w *
MemoBlock[iB].Cell[iiC].s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
MemoBlock[iB].InGate.net += // From InGate.
MemoBlock[iB].InGate.w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
MemoBlock[iB].InGate.net += // From OutGate.
MemoBlock[iB].InGate.w_OutGate[iiB].w * MemoBlock[iiB].OutGate.y;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].InGate.net += // From Cells.
MemoBlock[iB].InGate.w_Cell[iiB][iiC].w *
MemoBlock[iiB].Cell[iiC].y;
}
// net OutGate.
//MemoBlock[iB].OutGate.net=0; // Reset with Bias connection.
MemoBlock[iB].OutGate.net = MemoBlock[iB].OutGate.w_Bias.w;
for(int iI=0;iI<NbIn;iI++) // From Input.
MemoBlock[iB].OutGate.net += MemoBlock[iB].OutGate.w_In[iI].w *
((PatData.SeqList[Seq])[Pat])[iI];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
MemoBlock[iB].OutGate.net += // From InGate.
MemoBlock[iB].OutGate.w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
MemoBlock[iB].OutGate.net += // From OutGate.
MemoBlock[iB].OutGate.w_OutGate[iiB].w * MemoBlock[iiB].OutGate.y;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].OutGate.net += // From Cells.
MemoBlock[iB].OutGate.w_Cell[iiB][iiC].w *
MemoBlock[iiB].Cell[iiC].y;
}
#ifdef USE_FORGET_GATES
// net FgGate.
//MemoBlock[iB].FgGate.net=0; // Reset with Bias connection.
MemoBlock[iB].FgGate.net = MemoBlock[iB].FgGate.w_Bias.w;
for(int iI=0;iI<NbIn;iI++) // From Input.
MemoBlock[iB].FgGate.net += MemoBlock[iB].FgGate.w_In[iI].w *
((PatData.SeqList[Seq])[Pat])[iI];
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
MemoBlock[iB].FgGate.net += MemoBlock[iB].FgGate.w_s[iiC].w *
MemoBlock[iB].Cell[iiC].s;
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
MemoBlock[iB].FgGate.net += // From InGate.
MemoBlock[iB].FgGate.w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
MemoBlock[iB].FgGate.net += // From OutGate.
MemoBlock[iB].FgGate.w_OutGate[iiB].w * MemoBlock[iiB].OutGate.y;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].FgGate.net += // From Cells.
MemoBlock[iB].FgGate.w_Cell[iiB][iiC].w *
MemoBlock[iiB].Cell[iiC].y;
}
#ifdef RATIONAL_RECURSION
// Do a range check for the exp() in the recursion.
if(MemoBlock[iB].FgGate.net>709) MemoBlock[iB].FgGate.net=709;
if(MemoBlock[iB].FgGate.net<-709) MemoBlock[iB].FgGate.net=-709;
#endif
#endif
// net Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
MemoBlock[iB].Cell[iC].net=0;
#ifdef USE_CELL_BIAS
MemoBlock[iB].Cell[iC].net = MemoBlock[iB].Cell[iC].w_Bias.w;
#endif
for(int iI=0;iI<NbIn;iI++) // From Input.
MemoBlock[iB].Cell[iC].net += MemoBlock[iB].Cell[iC].w_In[iI].w *
((PatData.SeqList[Seq])[Pat])[iI];
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
MemoBlock[iB].Cell[iC].net += // From InGate.
MemoBlock[iB].Cell[iC].w_InGate[iiB].w * MemoBlock[iiB].InGate.y;
MemoBlock[iB].Cell[iC].net += // From OutGate.
MemoBlock[iB].Cell[iC].w_OutGate[iiB].w *
MemoBlock[iiB].OutGate.y;
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
MemoBlock[iB].Cell[iC].net += // From Cells.
MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].w *
MemoBlock[iiB].Cell[iiC].y;
}
}
} // End first loop over MemoBlocks.
// MemoBlocks (g,s,h,y).
for(int iB=0;iB<NbMemoBlocks;iB++) {
// y InGate.
MemoBlock[iB].InGate.y_t1 = MemoBlock[iB].InGate.y;
log_sig(MemoBlock[iB].InGate.net, MemoBlock[iB].InGate.y);
#ifdef USE_FORGET_GATES
// y FgGate.
#ifndef RATIONAL_RECURSION
Yfg(MemoBlock[iB].FgGate.net, MemoBlock[iB].FgGate.y);
#else
MemoBlock[iB].FgGate.y=0; // Not needed in this case (just for debug).
MemoBlock[iB].FgGate.exp_NegNet=exp(-MemoBlock[iB].FgGate.net);
#endif
#endif
// g,s,h,y Cell[iC].
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
MemoBlock[iB].Cell[iC].y_t1 = MemoBlock[iB].Cell[iC].y;
// G Input squashing.
G(MemoBlock[iB].Cell[iC].net, MemoBlock[iB].Cell[iC].g);
//MemoBlock[iB].Cell[iC].g = tanh(MemoBlock[iB].Cell[iC].net);
#ifdef USE_FORGET_GATES
// For the FgGate partials (and direct conection of the gates to s)
// we keep the last s.
MemoBlock[iB].Cell[iC].s_t1 = MemoBlock[iB].Cell[iC].s;
#ifndef RATIONAL_RECURSION
MemoBlock[iB].Cell[iC].s *= MemoBlock[iB].FgGate.y;
#else
MemoBlock[iB].Cell[iC].s = MemoBlock[iB].Cell[iC].s/
(1+fabs(MemoBlock[iB].Cell[iC].s)*MemoBlock[iB].FgGate.exp_NegNet);
#endif
#endif
//MemoBlock[iB].Cell[iC].s *= 0.9;
MemoBlock[iB].Cell[iC].s +=
MemoBlock[iB].Cell[iC].g * MemoBlock[iB].InGate.y;
// Do the overflow check for s (709 is max) in H() not here.
// H Output squashing.
H(MemoBlock[iB].Cell[iC].s, MemoBlock[iB].Cell[iC].h);
//MemoBlock[iB].Cell[iC].h = tanh(MemoBlock[iB].Cell[iC].s);
// Now we finish the OutGate net input, because it should see
// the actual s before it lets it out.
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
MemoBlock[iB].OutGate.net += MemoBlock[iB].OutGate.w_s[iiC].w *
MemoBlock[iB].Cell[iiC].s;
#endif
// y OutGate (the outher gates'ys are updated before the s calc).
MemoBlock[iB].OutGate.y_t1 = MemoBlock[iB].OutGate.y;
log_sig(MemoBlock[iB].OutGate.net, MemoBlock[iB].OutGate.y);
// Gating cell output.
MemoBlock[iB].Cell[iC].y = MemoBlock[iB].Cell[iC].h *
MemoBlock[iB].OutGate.y;
}
}
// Derivatives s_d.
for(int iB=BegBlockDeriv;iB<EndBlockDeriv;iB++) {
#ifndef LOG_LIKELIHOOD_GATES
log_sig_d(MemoBlock[iB].InGate.y,InGate_df); //For the s_d for InGate.
#else
InGate_df = 1;
#endif
#ifdef USE_FORGET_GATES
#if !defined(LOG_LIKELIHOOD_GATES) && !defined(RATIONAL_RECURSION)
Yfg_d(MemoBlock[iB].FgGate.y,FgGate_df); //For the s_d for FgGate.
#else
FgGate_df = 1;
#endif
#endif
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#if defined(USE_FORGET_GATES) && defined(RATIONAL_RECURSION)
frac_abs_s_NegNet = fabs(MemoBlock[iB].Cell[iC].s_t1)*
MemoBlock[iB].FgGate.exp_NegNet;
// For all.
frac_1_Sqr = 1/SQR(1+frac_abs_s_NegNet);
// For forget gates.
frac_Sqr_s_NegNet = MemoBlock[iB].Cell[iC].s_t1*frac_abs_s_NegNet;
#endif
#ifndef LOG_LIKELIHOOD_CELLS
// s_d for Cells.
#if defined(G_SIGMOID) || defined(G_TANH)
G_d(MemoBlock[iB].Cell[iC].g, g_d);
#elif defined(G_RATIONAL_FUNC) || defined(G_LINEAR)
G_d(g_d);
#endif
#else
g_d = 1;
#endif
g_d__y_in = g_d * MemoBlock[iB].InGate.y; // For all cells in block.
#ifdef USE_CELL_BIAS
Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_Bias,iB,1);
#endif
for(int iI=0;iI<NbIn;iI++)
Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_In[iI],iB,
((PatData.SeqList[Seq])[Pat])[iI]);
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_InGate[iiB],iB,
MemoBlock[iiB].InGate.y_t1);
Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_OutGate[iiB],iB,
MemoBlock[iiB].OutGate.y_t1);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
Update_Cell_s_d(MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC],iB,
MemoBlock[iiB].Cell[iiC].y_t1);
}
// s_d for InGate.
y_in_d__g = InGate_df * MemoBlock[iB].Cell[iC].g;
Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d,iB,1);
for(int iI=0;iI<NbIn;iI++)
Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d,iB,
((PatData.SeqList[Seq])[Pat])[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d,iB,
MemoBlock[iB].Cell[iiC].s_t1);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d,
iB,MemoBlock[iiB].InGate.y_t1);
Update_InGate_s_d(MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d,
iB,MemoBlock[iiB].OutGate.y_t1);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
Update_InGate_s_d
(MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d,
iB,MemoBlock[iiB].Cell[iiC].y_t1);
}
#ifdef USE_FORGET_GATES
// s_d for FgGate.
#ifndef RATIONAL_RECURSION
y__fg_d__s_t1 = FgGate_df * MemoBlock[iB].Cell[iC].s_t1;
#endif
Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d,iB,1);
for(int iI=0;iI<NbIn;iI++)
Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d,
iB,((PatData.SeqList[Seq])[Pat])[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d,iB,
MemoBlock[iB].Cell[iiC].s_t1);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
Update_FgGate_s_d(MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d,
iB,MemoBlock[iiB].InGate.y_t1);
Update_FgGate_s_d
(MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d,
iB,MemoBlock[iiB].OutGate.y_t1);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
Update_FgGate_s_d
(MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d,
iB,MemoBlock[iiB].Cell[iiC].y_t1);
}
#endif
} // End iC loop.
} // End iB loop.
// Update the Hidden units (y).
for(int iH=0;iH<NbHidden;iH++) {
//Hidden[iH].net=0; // Reset with bias.
Hidden[iH].net = Hidden[iH].w_Bias.w;
for(int iI=0;iI<NbIn;iI++) // From Input.
Hidden[iH].net += Hidden[iH].w_In[iI].w *
((PatData.SeqList[Seq])[Pat])[iI];
for(int iB=0;iB<NbMemoBlocks;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
Hidden[iH].net += Hidden[iH].w_Cell[iB][iC].w *
MemoBlock[iB].Cell[iC].y;
Hidden[iH].y = tanh(Hidden[iH].net);
}
// // Update the PredictNextIn units (y).
// for(int iP=0;iP<NbPredictNextIn;iP++)
// log_sig(PredictNextIn[iP].net, PredictNextIn[iP].y);
// // Update the NextIn units (y).
// if(NbPredictNextIn>0)
// for(int iO=0;iO<NbIn;iO++)
// log_sig(NextIn[iO].net, NextIn[iO].y);
// // Update the PredictClass units (y).
// for(int iP=0;iP<NbPredictClass;iP++)
// log_sig(PredictClass[iP].net, PredictClass[iP].y);
// // Update the PredictClassOut units (y).
// if(NbPredictClass>0)
// for(int iO=0;iO<NbOut;iO++)
// log_sig(PredictClassOut[iO].net, PredictClassOut[iO].y);
// Input to and Update the out units (net,y).
for(int iO=0;iO<NbOut;iO++) {
//Out[iO].net=0; // Reset with bias.
Out[iO].net=Out[iO].w_Bias.w;
#ifndef NO_IN_OUT_SHORTCUTS
for(int iI=0;iI<NbIn;iI++) // From Input. (input shortcuts)
Out[iO].net += Out[iO].w_In[iI].w*((PatData.SeqList[Seq])[Pat])[iI];
#endif
for(int iB=0;iB<NbMemoBlocks;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
Out[iO].net += Out[iO].w_Cell[iB][iC].w * MemoBlock[iB].Cell[iC].y;
for(int iH=0;iH<NbHidden;iH++) // From Hidden.
Out[iO].net += Out[iO].w_Hidden[iH].w * Hidden[iH].y;
#ifndef LINEAR_OUTPUT
log_sig(Out[iO].net, Out[iO].y);
#else
Out[iO].y = Out[iO].net;
#endif
}
}
void TLSTM::BackwardPass(unsigned int BegBlock, unsigned int EndBlock) {
// Calculate the errors (e,delta).
// e,delta out units.
for(int iO=0;iO<NbOut;iO++) {
if(SetStepTarget || (Pat==sTrainData.NbPat[Seq]-1)) {
Out[iO].e=((sTrainData.SeqList[Seq])[Pat])[NbIn+iO] - Out[iO].y;
//if(Pat<sTrainData.NbPat[Seq]-1) // Not the last Pat in Seq.
//Out[iO].e /= sTrainData.NbPat[Seq];
#if !defined(LOG_LIKELIHOOD_OUTPUT) && !defined(LINEAR_OUTPUT)
Out[iO].delta = log_sig_d(Out[iO].y,Out[iO].df) * Out[iO].e;
#else
Out[iO].delta = Out[iO].e; Out[iO].df=1;
#endif
} else {
// Keep track for local learning rate adaptation.
Out[iO].e=0; Out[iO].delta=0;
#if !defined(LOG_LIKELIHOOD_OUTPUT) && !defined(LINEAR_OUTPUT)
log_sig_d(Out[iO].y,Out[iO].df);
#else
Out[iO].df=1;
#endif
}
}
// e,delta Hidden units.
for(int iH=0;iH<NbHidden;iH++) {
Hidden[iH].e =0;
for(int iO=0;iO<NbOut;iO++) // From Out.
Hidden[iH].e += Out[iO].delta * Out[iO].w_Hidden[iH].w;
Hidden[iH].delta *= tanh_d(Hidden[iH].y,Hidden[iH].df) * Hidden[iH].e;
}
// // e,delta NextIn units.
// if(NbPredictNextIn>0)
// for(int iO=0;iO<NbIn;iO++) {
// if(Pat==sTrainData.NbPat[Seq]-1) NextIn[iO].e =0; // Last Pat in Seq.
// else NextIn[iO].e = // Difference to the next input.
// (((sTrainData.SeqList[Seq])[Pat+1])[iO] - NextIn[iO].y);
// NextIn[iO].delta = log_sig_d(NextIn[iO].y,NextIn[iO].df)
// * NextIn[iO].e;
// }
// // e,delta PredictNextIn units.
// for(int iP=0;iP<NbPredictNextIn;iP++) {
// PredictNextIn[iP].e =0;
// for(int iO=0;iO<NbIn;iO++) // Error From NextIn.
// PredictNextIn[iP].e += NextIn[iO].delta
// * NextIn[iO].w_PredictNextIn[iP].w;
// PredictNextIn[iP].delta *=
// log_sig_d(PredictNextIn[iP].y,PredictNextIn[iP].df)
// * PredictNextIn[iP].e;
// }
// // e,delta PredictClassOut units.
// if(NbPredictClass>0)
// for(int iO=0;iO<NbOut;iO++) {
// PredictClassOut[iO].e =
// (((sTrainData.SeqList[Seq])[sTrainData.NbPat[Seq]-1])[NbIn+iO]
// - PredictClassOut[iO].y);
// PredictClassOut[iO].delta =
// log_sig_d(PredictClassOut[iO].y,PredictClassOut[iO].df)
// * PredictClassOut[iO].e;
// }
// // e,delta PredictClass units.
// for(int iP=0;iP<NbPredictClass;iP++) {
// PredictClass[iP].e =0;
// for(int iO=0;iO<NbOut;iO++) // Error From PredictClassOut.
// PredictClass[iP].e += PredictClassOut[iO].delta *
// PredictClassOut[iO].w_PredictClass[iP].w;
// PredictClass[iP].delta =
// log_sig_d(PredictClass[iP].y,PredictClass[iP].df)
// * PredictClass[iP].e;
// }
// e_unscaled cells.
for(int iB=BegBlock;iB<EndBlock;iB++)
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
MemoBlock[iB].Cell[iC].e_unscaled=0;
for(int iO=0;iO<NbOut;iO++) // From regular Out units.
MemoBlock[iB].Cell[iC].e_unscaled += Out[iO].w_Cell[iB][iC].w
* Out[iO].delta;
// for(int iP=0;iP<NbPredictNextIn;iP++) // From PredictNextIn units.
// MemoBlock[iB].Cell[iC].e_unscaled +=
// PredictNextIn[iP].w_Cell[iB][iC].w *
// PredictNextIn[iP].delta / sTrainData.NbPat[Seq];
// for(int iP=0;iP<NbPredictClass;iP++) // From PredictClass units.
// MemoBlock[iB].Cell[iC].e_unscaled +=
// PredictClass[iP].w_Cell[iB][iC].w *
// PredictClass[iP].delta / sTrainData.NbPat[Seq];
}
// e cells (=e InGates, also for the FgGates).
for(int iB=BegBlock;iB<EndBlock;iB++)
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
#ifndef LOG_LIKELIHOOD_CELLS
MemoBlock[iB].Cell[iC].e = MemoBlock[iB].OutGate.y *
#if defined(H_SIGMOID) || defined(H_TANH)
H_d(MemoBlock[iB].Cell[iC].h)
#elif defined(H_RATIONAL_FUNC) || defined(H_LINEAR)
H_d()
#endif
* MemoBlock[iB].Cell[iC].e_unscaled;
// end of H_d (!! defines inside expression !!)
#else
MemoBlock[iB].Cell[iC].e = MemoBlock[iB].OutGate.y
* MemoBlock[iB].Cell[iC].e_unscaled;
#endif
// e,delta OutGates.
for(int iB=BegBlock;iB<EndBlock;iB++) {
MemoBlock[iB].OutGate.e=0;
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
MemoBlock[iB].OutGate.e += MemoBlock[iB].Cell[iC].h
* MemoBlock[iB].Cell[iC].e_unscaled;
#ifndef LOG_LIKELIHOOD_GATES
MemoBlock[iB].OutGate.delta =
log_sig_d(MemoBlock[iB].OutGate.y,MemoBlock[iB].OutGate.df)
* MemoBlock[iB].OutGate.e;
#else
MemoBlock[iB].OutGate.delta = MemoBlock[iB].OutGate.e;
MemoBlock[iB].OutGate.df = 1;
#endif
}
// Calcualte the local learning rate alpha and dw.
// Three layes in one time step.
// dw out units.
for(int iO=0;iO<NbOut;iO++) {
AdjustAlphaAndWeights(Out[iO].delta,Out[iO].df,1,Out[iO].w_Bias);
#ifndef NO_IN_OUT_SHORTCUTS
for(int iI=0;iI<NbIn;iI++) // From Input.
AdjustAlphaAndWeights(Out[iO].delta,Out[iO].df,
((sTrainData.SeqList[Seq])[Pat])[iI],
Out[iO].w_In[iI]);
#endif
for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
AdjustAlphaAndWeights(Out[iO].delta,Out[iO].df,
MemoBlock[iB].Cell[iC].y,
Out[iO].w_Cell[iB][iC]);
}
// dw Hidden units.
for(int iH=0;iH<NbHidden;iH++) {
AdjustAlphaAndWeights(Hidden[iH].delta,Hidden[iH].df,1,
Hidden[iH].w_Bias);
for(int iI=0;iI<NbIn;iI++) // From Input.
AdjustAlphaAndWeights(Hidden[iH].delta,Hidden[iH].df,
((sTrainData.SeqList[Seq])[Pat])[iI],
Hidden[iH].w_In[iI]);
for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
AdjustAlphaAndWeights(Hidden[iH].delta,Hidden[iH].df,
MemoBlock[iB].Cell[iC].y_t1,
Hidden[iH].w_Cell[iB][iC]);
}
// // dw NextIn units.
// if(NbPredictNextIn>0)
// for(int iO=0;iO<NbIn;iO++)
// for(int iP=0;iP<NbPredictNextIn;iP++)
// AdjustAlphaAndWeights(NextIn[iO].delta,NextIn[iO].df,
// PredictNextIn[iP].y,
// NextIn[iO].w_PredictNextIn[iP]);
// // dw PredicNextIn units.
// for(int iP=0;iP<NbPredictNextIn;iP++) {
// // PredictNextIn[iP].e *= AlphaPredict;
// for(int iI=0;iI<NbIn;iI++) // From Input.
// AdjustAlphaAndWeights(PredictNextIn[iP].delta,PredictNextIn[iP].df,
// ((sTrainData.SeqList[Seq])[Pat])[iI],
// PredictNextIn[iP].w_In[iI]);
// for(int iI=0;iI<NbOut;iI++) // From const class.
// AdjustAlphaAndWeights(PredictNextIn[iP].delta,PredictNextIn[iP].df,
// ((sTrainData.SeqList[Seq])
// [sTrainData.NbPat[Seq]-1])[NbIn+iI],
// PredictNextIn[iP].w_InClass[iI]);
// for(int iB=BegBlock;iB<EndBlock;iB++)
// for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
// AdjustAlphaAndWeights(PredictNextIn[iP].delta,PredictNextIn[iP].df,
// MemoBlock[iB].Cell[iC].y_t1,
// PredictNextIn[iP].w_Cell[iB][iC]);
// }
// // dw PredictClassOut units.
// if(NbPredictClass>0)
// for(int iO=0;iO<NbOut;iO++)
// for(int iP=0;iP<NbPredictClass;iP++)
// AdjustAlphaAndWeights(PredictClassOut[iO].delta,
// PredictClassOut[iO].df,
// PredictClass[iP].y,
// PredictClassOut[iO].w_PredictClass[iP]);
// // dw PredicClass units.
// for(int iP=0;iP<NbPredictClass;iP++)
// for(int iB=BegBlock;iB<EndBlock;iB++)
// for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
// AdjustAlphaAndWeights(PredictClass[iP].delta,PredictClass[iP].df,
// MemoBlock[iB].Cell[iC].y_t1,
// PredictClass[iP].w_Cell[iB][iC]);
// dw OutGate.
for(int iB=BegBlock;iB<EndBlock;iB++) {
AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
MemoBlock[iB].OutGate.df,
1,
MemoBlock[iB].OutGate.w_Bias);
for(int iI=0;iI<NbIn;iI++)
AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
MemoBlock[iB].OutGate.df,
((sTrainData.SeqList[Seq])[Pat])[iI],
MemoBlock[iB].OutGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
MemoBlock[iB].OutGate.df,
// OutGate see one step ahead,
// it used already s(t) not s(t-1).
MemoBlock[iB].Cell[iiC].s,
MemoBlock[iB].OutGate.w_s[iiC]);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
MemoBlock[iB].OutGate.df,
MemoBlock[iiB].InGate.y_t1,
MemoBlock[iB].OutGate.w_InGate[iiB]);
AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
MemoBlock[iB].OutGate.df,
MemoBlock[iiB].OutGate.y_t1,
MemoBlock[iB].OutGate.w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
AdjustAlphaAndWeights(MemoBlock[iB].OutGate.delta,
MemoBlock[iB].OutGate.df,
MemoBlock[iiB].Cell[iiC].y_t1,
MemoBlock[iB].OutGate.w_Cell[iiB][iiC]);
}
}
// dw cells.
for(int iB=BegBlock;iB<EndBlock;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef USE_CELL_BIAS
RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_Bias,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].w_Bias.alpha,
MemoBlock[iB].Cell[iC].w_Bias.h,
MemoBlock[iB].Cell[iC].w_Bias);
#endif
for(int iI=0;iI<NbIn;iI++)
RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_In[iI],
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].w_In[iI].alpha,
MemoBlock[iB].Cell[iC].w_In[iI].h,
MemoBlock[iB].Cell[iC].w_In[iI]);
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_InGate[iiB],
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].w_InGate[iiB].alpha,
MemoBlock[iB].Cell[iC].w_InGate[iiB].h,
MemoBlock[iB].Cell[iC].w_InGate[iiB]);
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_OutGate[iiB],
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].w_OutGate[iiB].alpha,
MemoBlock[iB].Cell[iC].w_OutGate[iiB].h,
MemoBlock[iB].Cell[iC].w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC],
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].alpha,
MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].h,
MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC]);
}
}
}
// dw InGate.
// Here we could do two runs. One to update alpha and
// than one for the weights, otherwise we get changing alphas during one
// weight update, ..ok.
for(int iB=BegBlock;iB<EndBlock;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_InGate_Bias.alpha,
MemoBlock[iB].Cell[iC].s_d_InGate_Bias.h,
MemoBlock[iB].InGate.w_Bias);
for(int iI=0;iI<NbIn;iI++)
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].alpha,
MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].h,
MemoBlock[iB].InGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].alpha,
MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].h,
MemoBlock[iB].InGate.w_s[iiC]);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].alpha,
MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].h,
MemoBlock[iB].InGate.w_InGate[iiB]);
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].alpha,
MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].h,
MemoBlock[iB].InGate.w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].alpha,
MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].h,
MemoBlock[iB].InGate.w_Cell[iiB][iiC]);
}
}
}
#ifdef USE_FORGET_GATES
// dw FgGate.
for(int iB=BegBlock;iB<EndBlock;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
RTRLAdjustAlphaAndWeights(MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.alpha,
MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.h,
MemoBlock[iB].FgGate.w_Bias);
for(int iI=0;iI<NbIn;iI++)
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].alpha,
MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].h,
MemoBlock[iB].FgGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].alpha,
MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].h,
MemoBlock[iB].FgGate.w_s[iiC]);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].alpha,
MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].h,
MemoBlock[iB].FgGate.w_InGate[iiB]);
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].alpha,
MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].h,
MemoBlock[iB].FgGate.w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
RTRLAdjustAlphaAndWeights
(MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d,
MemoBlock[iB].Cell[iC].e,
MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].alpha,
MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].h,
MemoBlock[iB].FgGate.w_Cell[iiB][iiC]);
}
}
}
#endif
}
#ifdef UPDATE_WEIGHTS_AFTER_SEQ
void TLSTM::ExecuteWeightChanges(unsigned int BegBlock,
unsigned int EndBlock) {
// w out units.
for(int iO=0;iO<NbOut;iO++) {
ExecuteWeightChange(Out[iO].w_Bias); // From Bias.
#ifndef NO_IN_OUT_SHORTCUTS
for(int iI=0;iI<NbIn;iI++) // From Input.
ExecuteWeightChange(Out[iO].w_In[iI]);
#endif
for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
ExecuteWeightChange(Out[iO].w_Cell[iB][iC]);
}
// w Hidden units.
for(int iH=0;iH<NbHidden;iH++) {
ExecuteWeightChange(Hidden[iH].w_Bias); // From Bias.
for(int iI=0;iI<NbIn;iI++) // From Input.
ExecuteWeightChange(Hidden[iH].w_In[iI]);
for(int iB=BegBlock;iB<EndBlock;iB++) // From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++)
ExecuteWeightChange(Hidden[iH].w_Cell[iB][iC]);
}
// w OutGate.
for(int iB=BegBlock;iB<EndBlock;iB++) {
ExecuteWeightChange(MemoBlock[iB].OutGate.w_Bias); // From Bias.
for(int iI=0;iI<NbIn;iI++)
ExecuteWeightChange(MemoBlock[iB].OutGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
ExecuteWeightChange(MemoBlock[iB].OutGate.w_s[iiC]);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
ExecuteWeightChange(MemoBlock[iB].OutGate.w_InGate[iiB]);
ExecuteWeightChange(MemoBlock[iB].OutGate.w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
ExecuteWeightChange(MemoBlock[iB].OutGate.w_Cell[iiB][iiC]);
}
}
// w cells.
for(int iB=BegBlock;iB<EndBlock;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
#ifdef USE_CELL_BIAS
ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_Bias); // From Bias.
#endif
for(int iI=0;iI<NbIn;iI++)
ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_In[iI]);
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
if(!InternalBlockConect && (iiB==iB)) continue;
#ifdef CONNECT_TO_GATES
ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_InGate[iiB]);
ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
ExecuteWeightChange(MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC]);
}
}
}
// w InGate.
for(int iB=BegBlock;iB<EndBlock;iB++) {
ExecuteWeightChange(MemoBlock[iB].InGate.w_Bias); // From Bias.
for(int iI=0;iI<NbIn;iI++)
ExecuteWeightChange(MemoBlock[iB].InGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
ExecuteWeightChange(MemoBlock[iB].InGate.w_s[iiC]);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
ExecuteWeightChange(MemoBlock[iB].InGate.w_InGate[iiB]);
ExecuteWeightChange(MemoBlock[iB].InGate.w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
ExecuteWeightChange(MemoBlock[iB].InGate.w_Cell[iiB][iiC]);
}
}
#ifdef USE_FORGET_GATES
// w FgGate.
for(int iB=BegBlock;iB<EndBlock;iB++) {
ExecuteWeightChange(MemoBlock[iB].FgGate.w_Bias); // From Bias.
for(int iI=0;iI<NbIn;iI++)
ExecuteWeightChange(MemoBlock[iB].FgGate.w_In[iI]);
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
ExecuteWeightChange(MemoBlock[iB].FgGate.w_s[iiC]);
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
ExecuteWeightChange(MemoBlock[iB].FgGate.w_InGate[iiB]);
ExecuteWeightChange(MemoBlock[iB].FgGate.w_OutGate[iiB]);
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
ExecuteWeightChange(MemoBlock[iB].FgGate.w_Cell[iiB][iiC]);
}
}
#endif
}
#endif
void TLSTM::PatStatistics(struct TPatData &PatData) {
// Next-In PredictNextIn statistics.
// MSEPat=0;
// if(NbPredictNextIn>0)
// for(int iO=0;iO<NbIn;iO++) // Error From NextIn.
// MSEPat += NextIn[iO].e * NextIn[iO].e;
// if(NbPredictClass>0)
// for(int iO=0;iO<NbOut;iO++) // Error From PredictClassOut.
// MSEPat += PredictClassOut[iO].e * PredictClassOut[iO].e;
// if(!SetStepTarget && NbPredictOut) MSEPat /= NbPredictOut;
// else {
// // Find the first two winners for RebberGrammar.
// // Find correct class.
// while(((PatData.SeqList[Seq])[Pat])[NbIn+FirstClassNb]!=1)
// FirstClassNb++; // At least one 1 should be there.
// SecondClassNb = FirstClassNb;
// do {
// SecondClassNb++; // Second winner.
// if(SecondClassNb>=NbOut) {
// SecondClassNb=FirstClassNb; break; } // No second.
// } while(((PatData.SeqList[Seq])[Pat])[NbIn+SecondClassNb]!=1);
// // Find winner and second.
// for(int iO=1;iO<NbOut;iO++)
// if(Out[iO].y>Out[SecondWinnerNb].y)
// if(Out[iO].y>Out[FirstWinnerNb].y) {
// SecondWinnerNb=FirstWinnerNb; FirstWinnerNb=iO;
// } else SecondWinnerNb=iO;
// if(FirstClassNb==SecondClassNb) {
// if(FirstWinnerNb!=FirstClassNb) PatWrong++;
// } else if((FirstWinnerNb!=FirstClassNb || SecondWinnerNb!=SecondClassNb)&&
// (FirstWinnerNb!=SecondClassNb || SecondWinnerNb!=FirstClassNb))
// PatWrong++;
// Patten is wrongly classified when error for at least one out-unit
// is over MaxMSE.
bool ThisPatWrong=false;
for(int iO=0;iO<NbOut;iO++)
if(fabs(Out[iO].e)>MaxMSE) {ThisPatWrong=true; break;}
if(ThisPatWrong) PatWrong++;
#ifdef DO_ONLINE_PAT_PRODUCTION
else PatCorrect++;
#endif
//#ifndef DO_ONLINE_PAT_PRODUCTION
// MSE
MSEPat=0;
for(int iO=0;iO<NbOut;iO++)MSEPat+=fabs(Out[iO].e);//Out[iO].e * Out[iO].e;
MSEPat /= NbOut;
// } // End else.
//MSEPat = sqrt(MSEPat); // Or just take the absolute error.
MSESeq += MSEPat;
//#endif
// // //ddd
// cout <<Epo << "-" << Seq << "-" << SeqOnline
// << "-" << Pat << " : ";
// for(int Val=0;Val<PatData.NbVal;Val++)
// cout << ((PatData.SeqList[Seq])[Pat])[Val] << " ";
// cout << "->" << PatCorrect << "(" << PatWrong << ")"
// << "(" << Out[0].y<<" -> "<< Out[0].e << ")"
// << " MSEPat:" << MSEPat
// << " MSESeq:" << MSESeq
// << " MSEEpo:" << MSEEpo;
// cout << "\n"; cout.flush();
////if(SeqOnline>=2) exit(1);
//ddd
}
void TLSTM::SeqStatistics(struct TPatData &PatData) {
#ifndef DO_ONLINE_PAT_PRODUCTION
/*
// Find correct class.
ClassNb=0;
while(((PatData.SeqList[Seq])[PatData.NbPat[Seq]-1])
[NbIn+ClassNb]!=1) ClassNb++;
// Find winner.
WinnerNb=0;
for(int iO=1;iO<NbOut;iO++)
if(Out[iO].y>Out[WinnerNb].y) WinnerNb=iO;
// Neterror.
if(WinnerNb!=ClassNb) ClassesWrong++;
#ifdef DEBUG
cerr << WinnerNb << " " << ClassNb << endl;
#endif
// MSE
if((NbPredictNextIn>0) || (NbPredictClass>0))
MSESeq /= PatData.NbPat[Seq]; // Last pattern for free.
else {
for(int iO=0;iO<NbOut;iO++) MSESeq += Out[iO].e * Out[iO].e;
MSESeq /= NbOut; MSESeq = sqrt(MSESeq);
}
*/
ClassesWrong+=PatWrong;//if(PatWrong) ClassesWrong++;
#endif
if(SetStepTarget) MSESeq /= PatData.NbPat[Seq]; MSEEpo += MSESeq;
}
void TLSTM::EpoStatistics(struct TPatData &PatData) {
#ifdef DO_ONLINE_PAT_PRODUCTION
if(SetStepTarget) PatCorrect/=PatData.NbSeq;
#ifdef BUFFER_EPO_LOG
TrainErrEpoLogBuf.AddValue(Epo, PatCorrect);
#else
WriteLogfileEpo(ERRORLOGFILE,PatCorrect);
#endif
if(PatCorrect>MaxPatCorrectTrain) MaxPatCorrectTrain=PatCorrect;
#else
#ifdef BUFFER_EPO_LOG
TrainErrEpoLogBuf.AddValue(Epo, ClassesWrong);
#else
WriteLogfileEpo(ERRORLOGFILE, ClassesWrong);
#endif
//ClassesWrongRel = ClassesWrong/PatData.NbSeq;
// Alpha = AlphaBase + AlphaError * ClassesWrongRel;
// Record the dLogError to decide when to add an other MemoBlock.
if(GrowNet) {
NbLogedErr++;
unsigned int devisor;
if(NbLogedErr>LogErrRecSize) devisor = LogErrRecSize;
else devisor = NbLogedErr;
LogErrRecMean = ((devisor-1)*LogErrRecMean+ClassesWrong)/devisor;
if(NbLogedErr>PartLogErrRecSize) devisor = PartLogErrRecSize;
else devisor = NbLogedErr;
PartLogErrRecMean =((devisor-1)*PartLogErrRecMean+ClassesWrong)/devisor;
if((NbLogedErr>LogErrRecSize) && (NbLogedErr>LogErrRecSize))
// Decide when to add an other MemoBlock.
if(PartLogErrRecMean>=LogErrRecMean) {
// Add memoblock, all same size.
if(GrowFirstOrder)
AddMemoBlock(-1,-1,MemoBlock[0].MemoBlockSize);
else // Grow cascate or fully connected.
AddMemoBlock(0,-1,MemoBlock[0].MemoBlockSize);
LogErrRecMean=0; PartLogErrRecMean=0; NbLogedErr=0;
}
WriteLogfileEpo(GROWLOGFILE, LogErrRecMean, PartLogErrRecMean);
}
#endif
// MSE for the training has to be compleated before a test is run.
#ifdef DO_ONLINE_PAT_PRODUCTION
MSEEpo /= (SeqOnline+1);
#else
MSEEpo /= PatData.NbSeq;
#endif
#ifdef BUFFER_EPO_LOG
TrainMSEEpoLogBuf.AddValue(Epo, MSEEpo);
#else
WriteLogfileEpo(MSELOGFILE, MSEEpo);
#endif
// Some more statistics.
#ifdef DO_WEIGHT_STATISTICS
#ifdef DO_ONLINE_PAT_PRODUCTION
WeightMean/=PatCount*NbWeights;
#else
WeightMean/=PatData.NbPatTotal*NbWeights;
#endif
//AlphaMean/=PatData.NbPatTotal*NbWeights;
//AlphaStd/=PatData.NbPatTotal*NbWeights;
//AlphaStd-=SQR(AlphaMean); AlphaStd=sqrt(AlphaStd);
WriteLogfileEpo(WEIGHTMEAN_LOGFILE, WeightMean);
//WriteLogfileEpo(ALPHAMEAN_LOGFILE, AlphaMean, AlphaStd);
WeightMean=0; AlphaMean=0; AlphaStd=0;
#endif
// Test the net on the Test-set. At the end not to disturb statistics.
if(TestEach_Epo>0) {
if((unsigned int)fmod(Epo, TestEach_Epo) == 0) Test(); }
#ifdef DO_ONLINE_PAT_PRODUCTION
else {
if(PatCorrect==MaxPatCorrectTrain && // Better in training.
LastTestEpo+TestMaxEach_Epo<Epo) { // Not test too much.
Test();
// Write the weight file for good test results.
// if(PatCorrectTest>500) { //ddd
// sprintf(cBuf, "w.Test%d.%d-%d",
// Tri,Epo,(unsigned int)PatCorrectTest);
// WriteWeightFile(cBuf);
// }
}
}
#endif
}
unsigned int TLSTM::Test() {
// cout << "Test...." << endl; //ddd
if (TestEpochs==0) return 0;
LastTestEpo=Epo;
unsigned int ClassesWrongTmp = ClassesWrong; // Save the train value.
double PatCorrectTmp = PatCorrect; // Save the train value.
unsigned int SeqOnlineTmp = SeqOnline;
unsigned int SeqOnlinePatCountTmp = SeqOnlinePatCount;
double MSEEpoTmp = MSEEpo;
ClassesWrong=0; TestMSEEpo=0; PatCorrectTest=0;
for(TestEpo=0;TestEpo<TestEpochs;TestEpo++) { // Test Epo loop.
PatCorrect=0; SeqOnline=0; SeqOnlinePatCount=0; MSEEpo=0;
#ifdef DO_ONLINE_PAT_PRODUCTION
// This Seq will be overwritten, we init the Nb variables here.
ONLINE_PAT_FUNCNAME(sTestData, true);
#endif
for(Seq=0;Seq<sTestData.NbSeq;FreezeSeqLoop ? Seq=0 : Seq++) {//Seq loop.
MSESeq=0; PatWrong=0;
#ifndef DO_ONLINE_PAT_PRODUCTION
ResetNet();
#else
if(!SeqOnline) ResetNet(); //For sequential online
#endif
#ifdef DO_ONLINE_PAT_PRODUCTION
#ifdef NO_RESET_AFTER_ONLINE_SEQ
if(SeqOnline) ONLINE_PAT_FUNCNAME(sTestData, false);
else ONLINE_PAT_FUNCNAME(sTestData, true);
#else
ONLINE_PAT_FUNCNAME(sTestData, true);
#endif
#endif
for(Pat=0;Pat<sTestData.NbPat[Seq];Pat++) {
ForwardPass(sTestData,0,0); // Pat loop, don't calc s_d.
// Calc output units' error instead of doing the Backpass.
if((Pat==sTestData.NbPat[Seq]-1) || SetStepTarget ||
(NbPredictNextIn>0) || (NbPredictClass>0)) {
for(int iO=0;iO<NbOut;iO++)
Out[iO].e=((sTestData.SeqList[Seq])[Pat])[NbIn+iO] - Out[iO].y;
PatStatistics(sTestData);
//ddd
// cout <<Epo<<"-"<<TestEpo<<"-"<<Seq<<"-"
// <<SeqOnline<<"-"<<Pat<<":";
// for(int Val=0;Val<sTestData.NbVal;Val++)
// cout << ((sTestData.SeqList[Seq])[Pat])[Val] << " ";
// cout << "->" << PatCorrect << "(" << PatWrong << ")"
// << "PCT:" << PatCorrectTest
// << "MSEPat:" << MSEPat << "TestMSEEpo:" << TestMSEEpo;
// cout << "\n"; //cout.flush();
//ddd
//if(/*Epo<MaxEpochs && StopLern /*&& Seq<10*/) { //ddd
if(OutputDebug && !TestEpo) { Dump_Inner_States(); //ddd
//sprintf(cBuf,"DumpAll.log.%d-%d-%d-%d-%d",//ddd
// Tri,Epo,Seq,SeqOnline,Pat);//ddd
//DumpAll(cBuf); //ddd
} //ddd
#ifdef DO_ONLINE_PAT_PRODUCTION
if(SetStepTarget) {
if(!PatWrong)
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
ONLINE_PAT_FUNCNAME(sTestData, false);
#else
; // Generate seq only before Pat loop.
#endif
else break; // PatWrong.
//if(ReberGrammarState==-1) ResetNet();
}
#endif
}
// if(OutputDebug/*Epo<MaxEpochs && StopLern /*&& Seq<10*/) { //ddd
// Dump_Inner_States(); //ddd
// sprintf(cBuf,"DumpAll.log.%d-%d-%d-%d-%d",//ddd
// Tri,Epo,Seq,SeqOnline,Pat);//ddd
// DumpAll(cBuf); //ddd
// } //ddd
} // End Pat loop.
SeqStatistics(sTestData);
#ifdef DO_ONLINE_PAT_PRODUCTION
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
if(!SetStepTarget)
#endif
if(!PatWrong) {
SeqOnline++;
if(PatCorrect>=MaxOnlineStreamLengthTest) break;
} else break;
#endif
} // End Seq loop.
TestMSEEpo += MSEEpo/(SeqOnline+1);
PatCorrectTest+=PatCorrect;
} // End TestEpochs loop.
//cout << "End of Test." << endl; //ddd
TestMSEEpo /= TestEpochs;
#ifdef DO_ONLINE_PAT_PRODUCTION
PatCorrectTest/=sTestData.NbSeq*TestEpochs;
//PatCorrectTest=PatCorrect; // Remember the test value.
#ifdef BUFFER_EPO_LOG
TestErrEpoLogBuf.AddValue(Epo, PatCorrectTest);
TestMSEEpoLogBuf.AddValue(Epo, TestMSEEpo);
#else
WriteLogfileEpo(TEST_ERRORLOGFILE,PatCorrectTest);
WriteLogfileEpo(TEST_MSELOGFILE,TestMSEEpo);
#endif
#else // We leave out the TestEpochs for the non online case.
TestClassesWrong = ClassesWrong; // Remember the test value.
#ifdef BUFFER_EPO_LOG
TestErrEpoLogBuf.AddValue(Epo, TestClassesWrong);
#else
WriteLogfileEpo(TEST_ERRORLOGFILE, TestClassesWrong);
#endif
#endif
// Reconstruct the train values.
ClassesWrong = ClassesWrongTmp; // Reconstruct the train value for stop.
PatCorrect = PatCorrectTmp; // Reconstruct the train value for stop.
SeqOnline = SeqOnlineTmp;
SeqOnlinePatCount = SeqOnlinePatCountTmp;
MSEEpo = MSEEpoTmp;
#ifdef DO_ONLINE_PAT_PRODUCTION
return((unsigned int) PatCorrectTest);
#else
return TestClassesWrong;
#endif
}
char TLSTM::WriteWeightFile(char *FileName) {
if (OpenFile(FileName, ios::out)) return 1;
pFile->precision(PRECISION);
WriteWeightStream(pFile,(void (*)(class iostream *, double &))&d2s,true);
return CloseFile();
}
char TLSTM::LoadWeightFile(char *FileName) {
if (OpenFile(FileName, ios::in)) return 1;
WriteWeightStream(pFile,(void (*)(class iostream *, double &))&s2d,false);
return CloseFile();
}
char TLSTM::DumpAll(char *FileName) {
if (OpenFile(FileName, ios::out)) return 1;
pFile->precision(PRECISION);
*pFile << "weights\n\n";
WriteWeightStream(pFile,(void (*)(class iostream *, double &))&d2s,true);
WriteNetStream(pFile,sTrainData);
return CloseFile();
}
char TLSTM::WriteWeightStream(iostream *s,
void(*f)(iostream *s, double &d),
bool WriteWhiteSpace) {
double zero=0; // Dummy zeros to fill the weight matrix with not exist. w..
// Freds Weight Matrix(row=to;column=from):
// 1xBias(1 bias cell to all),(s),In,Hidden,
// MemoBlocks(Ingate,Outgate,FgGate,cells),Out.
// Zero rows or columns are left out.
// To Bias. (nothing)
// To In. (nothing)
// To s. (nothing)
// To Hidden. (!!!nothing from the Gates)
// Without CONNECT_TO_GATES defined these weights are not printed.
// Forget gates never have outgoing connections, so never printed.
for(int iH=0;iH<NbHidden;iH++) {
f(s,Hidden[iH].w_Bias.w); if(WriteWhiteSpace) *s << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) { // From In.
f(s,Hidden[iH].w_In[iI].w); if(WriteWhiteSpace) *s << " "; }
for(int iB=0;iB<NbMemoBlocks;iB++) {// From Blocks.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // From Cells.
#ifdef CONNECT_TO_GATES
// Add zeros for the missing connections from the gates.
f(s,zero); if(WriteWhiteSpace) *s << " "; // From InGate.
f(s,zero); if(WriteWhiteSpace) *s << " "; // From OutGate.
#ifdef USE_FORGET_GATES
f(s,zero); if(WriteWhiteSpace) *s << " "; // From FgGate.
#endif
#endif
f(s,Hidden[iH].w_Cell[iB][iC].w);
if(WriteWhiteSpace) *s << " "; }
}
} //if(WriteWhiteSpace) *s << "\n";
// To MemoBlocks.
for(int iB=0;iB<NbMemoBlocks;iB++) {
// To InGate.
f(s,MemoBlock[iB].InGate.w_Bias.w);
if(WriteWhiteSpace) *s << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) {
f(s,MemoBlock[iB].InGate.w_In[iI].w);
if(WriteWhiteSpace) *s << " "; } // From In.
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) { // From s.
f(s,MemoBlock[iB].InGate.w_s[iiC].w);
if(WriteWhiteSpace) *s << " "; }
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
f(s,MemoBlock[iB].InGate.w_InGate[iiB].w);
if(WriteWhiteSpace) *s << " "; // From InGate.
f(s,MemoBlock[iB].InGate.w_OutGate[iiB].w);
if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
f(s,MemoBlock[iB].InGate.w_Cell[iiB][iiC].w);
if(WriteWhiteSpace) *s << " "; }// From Cells.
} if(WriteWhiteSpace) *s << "\n";
// OutGate weights.
f(s,MemoBlock[iB].OutGate.w_Bias.w);
if(WriteWhiteSpace) *s << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) {
f(s,MemoBlock[iB].OutGate.w_In[iI].w);
if(WriteWhiteSpace) *s << " "; } // From In.
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) { // From s.
f(s,MemoBlock[iB].OutGate.w_s[iiC].w);
if(WriteWhiteSpace) *s << " "; }
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
f(s,MemoBlock[iB].OutGate.w_InGate[iiB].w);
if(WriteWhiteSpace) *s << " "; // From InGate.
f(s,MemoBlock[iB].OutGate.w_OutGate[iiB].w);
if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
f(s,MemoBlock[iB].OutGate.w_Cell[iiB][iiC].w);
if(WriteWhiteSpace) *s << " "; } // From Cells.
} if(WriteWhiteSpace) *s << "\n";
#ifdef USE_FORGET_GATES
// To FgGate.
f(s,MemoBlock[iB].FgGate.w_Bias.w);
if(WriteWhiteSpace) *s << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) {
f(s,MemoBlock[iB].FgGate.w_In[iI].w);
if(WriteWhiteSpace) *s << " "; } // From In.
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) { // From s.
f(s,MemoBlock[iB].FgGate.w_s[iiC].w);
if(WriteWhiteSpace) *s << " "; }
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
f(s,MemoBlock[iB].FgGate.w_InGate[iiB].w);
if(WriteWhiteSpace) *s << " "; // From InGate.
f(s,MemoBlock[iB].FgGate.w_OutGate[iiB].w);
if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
f(s,MemoBlock[iB].FgGate.w_Cell[iiB][iiC].w);
if(WriteWhiteSpace) *s << " "; }// From Cells.
} if(WriteWhiteSpace) *s << "\n";
#endif
// To Cells.
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
// Cell weights.
#ifdef USE_CELL_BIAS
f(s,MemoBlock[iB].Cell[iC].w_Bias.w);
#else
f(s,zero);
#endif
if(WriteWhiteSpace) *s << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) {
f(s,MemoBlock[iB].Cell[iC].w_In[iI].w);
if(WriteWhiteSpace) *s << " "; }// From In.
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {// From Blocks.
#ifdef CONNECT_TO_GATES
f(s,MemoBlock[iB].Cell[iC].w_InGate[iiB].w);
if(WriteWhiteSpace) *s << " ";// From InGate.
f(s,MemoBlock[iB].Cell[iC].w_OutGate[iiB].w);
if(WriteWhiteSpace) *s << " ";// From OutGate.
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
f(s,MemoBlock[iB].Cell[iC].w_Cell[iiB][iiC].w);
if(WriteWhiteSpace) *s << " "; } //From Cells
} if(WriteWhiteSpace) *s << "\n";
} //if(WriteWhiteSpace) *s << "\n";
}
// Out units' weights.
for(int iO=0;iO<NbOut;iO++) {
//if(WriteWhiteSpace) *s << "\n";
f(s,Out[iO].w_Bias.w); if(WriteWhiteSpace) *s << " "; // From Bias.
#ifndef NO_IN_OUT_SHORTCUTS
for(int iI=0;iI<NbIn;iI++) {
f(s,Out[iO].w_In[iI].w); if(WriteWhiteSpace) *s << " "; }// From In.
#endif
for(int iB=0;iB<NbMemoBlocks;iB++) { // From Blocks.
#ifdef CONNECT_TO_GATES
// Add zeros for the missing connections to the gates.
f(s,zero); if(WriteWhiteSpace) *s << " "; // From InGate.
f(s,zero); if(WriteWhiteSpace) *s << " "; // From OutGate.
#endif
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
f(s,Out[iO].w_Cell[iB][iC].w);
if(WriteWhiteSpace) *s << " "; }//From Cells
}if(WriteWhiteSpace) *s << "\n";
}
if(WriteWhiteSpace) *s << "\n"; // For matrix2X tool.
}
void TLSTM::DisplayNet(struct TPatData &PatData) {
WriteNetStream(NULL,PatData);
}
void TLSTM::WriteNetStream(iostream *s, struct TPatData &PatData) {
strstream DisplayBuf;
// Flags for DisplayBuf
DisplayBuf.precision(PRECISION);
cWidth = cPrecision+6;
if(!s) {
DisplayBuf.fill(' ');
DisplayBuf.setf(ios::showpos);
DisplayBuf.setf(ios::left, ios::adjustfield);
// Start writing into the buffer.
DisplayBuf << " Tri:" << Tri << "/" << NbTrials
<< " Epo:" << Epo << "/" << MaxEpochs
<< " Seq:" << Seq << "/" << PatData.NbSeq
<< " Pat:" << Pat << "/" << PatData.NbPat[Seq] << "\n"
<< " ClassesWrong:" << ClassesWrong
<< " PatWrong: " << PatWrong
<< " MSEEpo: " << MSEEpo
<< " MSESeq: " << setw(cWidth) << MSESeq
<< " MSEPat: " << setw(cWidth) << MSEPat
<< "\n";
if(Pat>PatData.NbPat[Seq]-1) Pat=PatData.NbPat[Seq]-1;
DisplayBuf << " ";
for(int iO=0;iO<NbOut;iO++) // Targets
DisplayBuf << ((PatData.SeqList[Seq])[Pat])[NbIn+iO]
<< " ";
DisplayBuf << "\n";
}
DisplayBuf << "\nactivations\n\n"; // activations.
// No Hidden yet.
for(int iB=0;iB<NbMemoBlocks;iB++) {
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].InGate.y << " "; // InGate.y
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].OutGate.y << " "; // OutGate.y
#ifdef USE_FORGET_GATES
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].FgGate.y << " "; // FgGate.y
#endif
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.y
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].Cell[iC].y << " ";
}
DisplayBuf << "\n";
}
for(int iO=0;iO<NbOut;iO++) DisplayBuf << Out[iO].y << " "; // Out.y
DisplayBuf << "\n\ncellstates\n\n"; // cellstates.
for(int iB=0;iB<NbMemoBlocks;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.s
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].Cell[iC].s << " ";
}
DisplayBuf << "\n";
}
DisplayBuf << "\ng\n\n"; // g
for(int iB=0;iB<NbMemoBlocks;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.g
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].Cell[iC].g << " ";
}
DisplayBuf << "\n";
}
DisplayBuf << "\nh\n\n"; // h
for(int iB=0;iB<NbMemoBlocks;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cell.h
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].Cell[iC].h << " ";
}
DisplayBuf << "\n";
}
// Errors.
DisplayBuf << "\nouterror\n\n";
//for(int iO=0;iO<NbOut;iO++)
// DisplayBuf << Out[iO].e << " "; // Out.e
for(int iO=0;iO<NbOut;iO++)
DisplayBuf << Out[iO].delta << " "; // Out.delta
DisplayBuf << "\n\noutgateerror\n\n";
for(int iB=0;iB<NbMemoBlocks;iB++) {
//DisplayBuf << setw(cWidth);
//DisplayBuf << MemoBlock[iB].OutGate.e << " "; // OutGate.e
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].OutGate.delta; // OutGate.delta
DisplayBuf << "\n";
}
DisplayBuf << "\n\ncellerror\n\n";
for(int iB=0;iB<NbMemoBlocks;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cells.e_unscaled
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].Cell[iC].e_unscaled << " ";
}
DisplayBuf << "\n";
}
DisplayBuf << "\nstateerror\n\n";
for(int iB=0;iB<NbMemoBlocks;iB++) {
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) { // Cells.e
//DisplayBuf << setw(cWidth);
DisplayBuf << MemoBlock[iB].Cell[iC].e << " ";
}
DisplayBuf << "\n";
}
// s_d for the InGate.
DisplayBuf << "\ningatepartials\n\n";
for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate_Bias.s_d << " ";
for(int iI=0;iI<NbIn;iI++)
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate_In[iI].s_d << " ";
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate_s[iiC].s_d << " ";
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
DisplayBuf <<
MemoBlock[iB].Cell[iC].s_d_InGate_InGate[iiB].s_d << " ";
DisplayBuf <<
MemoBlock[iB].Cell[iC].s_d_InGate_OutGate[iiB].s_d << " ";
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
DisplayBuf
<< MemoBlock[iB].Cell[iC].s_d_InGate_Cell[iiB][iiC].s_d << " ";
}
DisplayBuf << "\n";
}
}
#ifdef USE_FORGET_GATES
// s_d for the FgGate.
DisplayBuf << "\nFgGatepartials\n\n";
for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_FgGate_Bias.s_d << " ";
for(int iI=0;iI<NbIn;iI++)
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_FgGate_In[iI].s_d << " ";
#ifdef CONNECT_GATES_TO_S
for(int iiC=0;iiC<MemoBlock[iB].MemoBlockSize;iiC++) // From s.
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_FgGate_s[iiC].s_d << " ";
#endif
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
DisplayBuf <<
MemoBlock[iB].Cell[iC].s_d_FgGate_InGate[iiB].s_d << " ";
DisplayBuf <<
MemoBlock[iB].Cell[iC].s_d_FgGate_OutGate[iiB].s_d << " ";
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
DisplayBuf
<< MemoBlock[iB].Cell[iC].s_d_FgGate_Cell[iiB][iiC].s_d << " ";
}
DisplayBuf << "\n";
}
}
#endif
// s_d for the cells.
DisplayBuf << "\ncellpartials\n\n";
for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_Bias << " ";
for(int iI=0;iI<NbIn;iI++)
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_In[iI] << " ";
for(int iiB=MemoBlock[iB].BegSrcBlockNb;
iiB<MemoBlock[iB].EndSrcBlockNb;iiB++) {
#ifdef CONNECT_TO_GATES
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_InGate[iiB] << " ";
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_OutGate[iiB] << " ";
#endif
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
DisplayBuf << MemoBlock[iB].Cell[iC].s_d_Cell[iiB][iiC] << " ";
}
DisplayBuf << "\n";
}
}
DisplayBuf << "\n";
if(!s) {
for(int iI=0;iI<NbIn;iI++) // In
DisplayBuf << ((PatData.SeqList[Seq])[Pat])[iI] << " ";
DisplayBuf << "\n\n";
}
// Print the buffer.
char* DisplayStr = DisplayBuf.str();
DisplayBuf << ends;
if(!s) { // Print to stdout.
printf("\033[0;H\007"); // position curser (0,0) (row,colm)
printf("\033[0J\007"); // Clear Below
cout << DisplayStr; cout.flush();
} else *s << DisplayStr;
free(DisplayStr);
//DisplayBuf.clear(); DisplayBuf.seekg(0); DisplayBuf.seekp(0);
}
void TLSTM::DisplayWeights() {
strstream WeightBuf; WeightBuf.precision(4);
WriteWeightStream(&WeightBuf,(void (*)(class iostream *, double &))&d2s,
true);
// Print the buffer.
WeightBuf << ends;
char *WeightBufStr = WeightBuf.str();
cout << WeightBufStr; free(WeightBufStr);
cout.flush(); // Finally flush all out (including the buffers).
//WeightBuf.clear(); WeightBuf.seekg(0); WeightBuf.seekp(0);
}
char TLSTM::Dump_Inner_States() {
#ifdef DO_ONLINE_PAT_PRODUCTION
// Log seq beginnings and endings.
if(!SetStepTarget) {
if(Pat==0) {
sprintf(cBuf,"SeqBegEnd.log.%d-%d-%d",Tri,Epo,Seq);
if(OpenFile(cBuf, ios::app)) return 1;
// ddd TempOrderEx6
// *pFile << SeqOnlinePatCount << " " << sTestData.NbClass[0] << "\n";
// *pFile << SeqOnlinePatCount+t1 << " " << XNotYt1 << "\n";
// *pFile << SeqOnlinePatCount+t2 << " " << XNotYt2 << "\n";
// *pFile << SeqOnlinePatCount+t3 << " " << XNotYt3 << "\n";
CloseFile();
}
}
#endif
// Dump the activation of the out units.
sprintf(cBuf,"%s.%d", OUT_LOGFILE,Tri);
if(OpenFile(cBuf, ios::app)) return 1;
pFile->precision(4);
#ifndef DO_ONLINE_PAT_PRODUCTION
*pFile << Pat << " ";
#else
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
if(SetStepTarget) *pFile << Pat << " ";
else *pFile << SeqOnlinePatCount++ << " ";
#else
if(SetStepTarget) *pFile << PatCorrect << " ";
#endif
#endif
for(int iO=0;iO<NbOut;iO++){ *pFile << Out[iO].y << " "; }
*pFile << "\n";
CloseFile(); // No return, no check here.
//
// Dump the cells' inner states s.
//sprintf(cBuf,"%s.%d-%d-%d", INNERSTATE_DUMP_LOGFILE,Tri,Epo,Seq);
//sprintf(cBuf,"%s.%d-%d", INNERSTATE_DUMP_LOGFILE,Tri,Seq);
sprintf(cBuf,"%s.%d", INNERSTATE_DUMP_LOGFILE,Tri);
//sprintf(cBuf,"%s.%d-%d", INNERSTATE_DUMP_LOGFILE,Tri,Epo);
if(OpenFile(cBuf, ios::app)) return 1;
pFile->precision(4);
#ifndef DO_ONLINE_PAT_PRODUCTION
*pFile << Pat << " ";
#else
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
if(SetStepTarget) *pFile << Pat << " ";
else *pFile << SeqOnlinePatCount++ << " ";
#else
if(SetStepTarget) *pFile << PatCorrect << " ";
#endif
#endif
for(int iB=0;iB<NbMemoBlocks;iB++)
for(int iC=0;iC<MemoBlock[iB].MemoBlockSize;iC++) {
*pFile << MemoBlock[iB].Cell[iC].s << " ";
}
*pFile << "\n";
#ifndef USE_FORGET_GATES
return CloseFile();
#else
bool FirstFileClose = CloseFile();
//sprintf(cBuf, "%s.%d-%d-%d",FGGATE_Y_LOGFILE,Tri,Seq);
//sprintf(cBuf, "%s.%d-%d",FGGATE_Y_LOGFILE,Tri,Epo,Seq);
sprintf(cBuf, "%s.%d",FGGATE_Y_LOGFILE,Tri);
//sprintf(cBuf, "%s.%d-%d",FGGATE_Y_LOGFILE,Tri,Epo);
if(OpenFile(cBuf, ios::app)) return 1;
pFile->precision(6);
#ifndef DO_ONLINE_PAT_PRODUCTION
*pFile << Pat << " ";
#else
#ifndef SETSTEPTARGET_BUT_COUNT_SEQUENCEWISE
if(SetStepTarget) *pFile << Pat << " ";
else *pFile << SeqOnlinePatCount << " ";
#else
if(SetStepTarget) *pFile << PatCorrect << " ";
#endif
#endif
for(int iB=0;iB<NbMemoBlocks;iB++)
#ifndef RATIONAL_RECURSION
*pFile << MemoBlock[iB].FgGate.y << " ";
#else
*pFile << MemoBlock[iB].FgGate.net << " ";
#endif
*pFile << "\n";
return (CloseFile() && FirstFileClose);
#endif
}
char TLSTM::DumpSomeAlphas() {
if (OpenFile("AlphaDump.log", ios::app)) return 1;
pFile->precision(4);
*pFile << Epo << " ";
// Alpha for the InGate.
for(int iC=0;iC<MemoBlock[3].MemoBlockSize;iC++) {
*pFile << MemoBlock[3].Cell[iC].s_d_InGate_Bias.alpha << " ";
for(int iI=0;iI<NbIn;iI++)
*pFile << MemoBlock[3].Cell[iC].s_d_InGate_In[iI].alpha << " ";
for(int iiB=2;iiB<3;iiB++) {
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++)
*pFile
<< MemoBlock[3].Cell[iC].s_d_InGate_Cell[iiB][iiC].alpha << " ";
}
}
// OutGate weights alphas.
*pFile << MemoBlock[3].OutGate.w_Bias.alpha;
*pFile << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) {
*pFile << MemoBlock[3].OutGate.w_In[iI].alpha;
*pFile << " "; } // From In.
// Cells.
for(int iC=0;iC<MemoBlock[3].MemoBlockSize;iC++) {
// Cell weights alphas.
*pFile << " "; // From Bias.
for(int iI=0;iI<NbIn;iI++) {
*pFile << MemoBlock[3].Cell[iC].w_In[iI].alpha;
*pFile << " "; }// From In.
for(int iiB=2;iiB<3;iiB++) {// From Blocks.
*pFile << MemoBlock[3].Cell[iC].w_InGate[iiB].alpha;
*pFile << " ";// From InGate.
*pFile << MemoBlock[3].Cell[iC].w_OutGate[iiB].alpha;
*pFile << " ";// From OutGate.
for(int iiC=0;iiC<MemoBlock[iiB].MemoBlockSize;iiC++) {
*pFile << MemoBlock[3].Cell[iC].w_Cell[iiB][iiC].alpha;
*pFile << " "; } //From Cells
}
}
*pFile << "\n";
return CloseFile();
}
char TLSTM::WriteOutLogFile(char *FileName) {
if (OpenFile(FileName, ios::app)) return 1;
*pFile << Pat << " ";
for(int iO=0;iO<NbOut;iO++) *pFile << Out[iO].y << " ";
*pFile << "\n";
return CloseFile();
}
void TLSTM::TopoCheck() {
cout << " "<< NbCells << " " << NbMemUnits << " " << NbWeights << endl;
for(int iB=0;iB<NbMemoBlocks;iB++) { // MemoBlocks
cout << iB << " : "
<< MemoBlock[iB].BegSrcBlockNb << "-"
<< MemoBlock[iB].EndSrcBlockNb << endl;
}
cout << FreezeEndBlock << endl;
}
void TLSTM::SetTopoStatistics() {
// Count the number of gates.
// For the input gates we count NbCells*Weights,
// so a weight to the gate is counted NbCell times.
unsigned int NbGates=NbMemoBlocks*2;
NbWeights=0; NbCells=0;
for(int iB=0;iB<NbMemoBlocks;iB++) NbCells+=MemoBlock[iB].MemoBlockSize;
NbMemUnits = NbCells+NbGates;
// Bias.
NbWeights += NbMemUnits+NbOut;
#ifndef USE_CELL_BIAS
NbWeights -= NbCells;
#endif
#ifdef CONNECT_GATES_TO_S
for(int iB=0;iB<NbMemoBlocks;iB++) {
NbWeights += MemoBlock[iB].MemoBlockSize *2;
#ifdef USE_FORGET_GATES
NbWeights += MemoBlock[iB].MemoBlockSize;
#endif
}
#endif
// hidden to out.
NbWeights += NbCells * NbOut;
// Start with fully connected memblocks then substract.
NbWeights += SQR(NbMemUnits);
#ifdef CONNECT_TO_GATES
NbWeights-=NbGates*NbMemUnits;
#endif
if(!InternalBlockConect)
for(int iB=0;iB<NbMemoBlocks;iB++)
NbWeights-=SQR(MemoBlock[iB].MemoBlockSize);
// Connetion from the input.
NbWeights+=(NbMemUnits+NbOut)*NbIn;
// Now count the weight to the InGate * NbCells as extra weights.
unsigned int NbWeightsToInGate = NbIn+NbCells+1; // 1 for bias.
#ifdef CONNECT_TO_GATES
NbWeightsToInGate += NbGates;
#endif
for(int iB=0;iB<NbMemoBlocks;iB++)
NbWeights+=(MemoBlock[iB].MemoBlockSize-1)*NbWeightsToInGate;
#ifdef USE_FORGET_GATES
// Same as to the InGates only that they are not yet counted at all.
for(int iB=0;iB<NbMemoBlocks;iB++)
NbWeights+=(MemoBlock[iB].MemoBlockSize)*NbWeightsToInGate;
#endif
}
char TLSTM::GetOnlineProblemStatistics() {
unsigned int NbCyl, CylLength;
if(OpenFile("ONLINE_PAT_FUNCNAME.log", ios::out)) return 1;
ONLINE_PAT_FUNCNAME(sTrainData, true);
NbCyl=0;CylLength=0;
while(NbCyl<0) {
ONLINE_PAT_FUNCNAME(sTrainData, false);
CylLength++;
// if(ReberGrammarState==0) { //ddd
// NbCyl++;
// *pFile << CylLength << "\n";
// CylLength=0;
// }
}
return CloseFile();
}
///////////////////main
main(int argc, char **argv)
{
//Global memory error management called by new
set_new_handler(MemoryFull); // __new_handler = &MemoryFull;
// Log stderr (cerr) into a file.
//dup2(open("cerr.log", O_WRONLY | O_CREAT | O_TRUNC, 0777),STDERR_FILENO);
int fdCerr = open("cerr.log", O_WRONLY | O_CREAT | O_TRUNC, 0666);
dup2(fdCerr, STDERR_FILENO);
// The User Interface (UI) is the only static object.
TLSTM LSTM;
return LSTM.Run();
// clean up
close(fdCerr);
}
- Raw text -