sdyna.h

Go to the documentation of this file.

/***************************************************************************
 *   Copyright (C) 2005 by Christophe GONZALES and Pierre-Henri WUILLEMIN  *
 *   {prenom.nom}_at_lip6.fr                                               *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/
// =========================================================================
#ifndef GUM_SDYNA_H
#define GUM_SDYNA_H
// =========================================================================
#include <agrum/agrum.h>
#include <agrum/multidim/instantiation.h>
// =========================================================================
#include <agrum/variables/discreteVariable.h>
// =========================================================================
#include <agrum/FMDP/SDyna/Strategies/IDecisionStrategy.h>
#include <agrum/FMDP/SDyna/Strategies/ILearningStrategy.h>
#include <agrum/FMDP/SDyna/Strategies/IPlanningStrategy.h>
#include <agrum/FMDP/decision/E_GreedyDecider.h>
#include <agrum/FMDP/decision/lazyDecider.h>
#include <agrum/FMDP/decision/randomDecider.h>
#include <agrum/FMDP/decision/statisticalLazyDecider.h>
#include <agrum/FMDP/fmdp.h>
#include <agrum/FMDP/learning/fmdpLearner.h>
#include <agrum/FMDP/learning/observation.h>
#include <agrum/FMDP/planning/actionSet.h>
#include <agrum/FMDP/planning/adaptiveRMaxPlaner.h>
#include <agrum/FMDP/planning/structuredPlaner.h>
// =========================================================================

namespace gum {

  class SDYNA {
    // ###################################################################
    // ###################################################################
    public:
    // ==========================================================================
    // ==========================================================================
    static SDYNA* spitiInstance(double attributeSelectionThreshold = 0.99,
                                double discountFactor = 0.9,
                                double epsilon = 1,
                                Idx    observationPhaseLenght = 100,
                                Idx    nbValueIterationStep = 10) {
      bool               actionReward = false;
      ILearningStrategy* ls = new FMDPLearner< CHI2TEST, CHI2TEST, ITILEARNER >(
         attributeSelectionThreshold, actionReward);
      IPlanningStrategy< double >* ps =
         StructuredPlaner< double >::sviInstance(discountFactor, epsilon);
      IDecisionStrategy* ds = new E_GreedyDecider();
      return new SDYNA(
         ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }

    // ==========================================================================
    // ==========================================================================
    static SDYNA* spimddiInstance(double attributeSelectionThreshold = 0.99,
                                  double similarityThreshold = 0.3,
                                  double discountFactor = 0.9,
                                  double epsilon = 1,
                                  Idx    observationPhaseLenght = 100,
                                  Idx    nbValueIterationStep = 10) {
      bool               actionReward = false;
      ILearningStrategy* ls = new FMDPLearner< GTEST, GTEST, IMDDILEARNER >(
         attributeSelectionThreshold, actionReward, similarityThreshold);
      IPlanningStrategy< double >* ps = StructuredPlaner< double >::spumddInstance(
         discountFactor, epsilon, false);
      IDecisionStrategy* ds = new E_GreedyDecider();
      return new SDYNA(ls,
                       ps,
                       ds,
                       observationPhaseLenght,
                       nbValueIterationStep,
                       actionReward,
                       false);
    }

    // ==========================================================================
    // ==========================================================================
    static SDYNA* RMaxMDDInstance(double attributeSelectionThreshold = 0.99,
                                  double similarityThreshold = 0.3,
                                  double discountFactor = 0.9,
                                  double epsilon = 1,
                                  Idx    observationPhaseLenght = 100,
                                  Idx    nbValueIterationStep = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls = new FMDPLearner< GTEST, GTEST, IMDDILEARNER >(
         attributeSelectionThreshold, actionReward, similarityThreshold);
      AdaptiveRMaxPlaner* rm = AdaptiveRMaxPlaner::ReducedAndOrderedInstance(
         ls, discountFactor, epsilon);
      IPlanningStrategy< double >* ps = rm;
      IDecisionStrategy*           ds = rm;
      return new SDYNA(
         ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }

    // ==========================================================================
    // ==========================================================================
    static SDYNA* RMaxTreeInstance(double attributeSelectionThreshold = 0.99,
                                   double discountFactor = 0.9,
                                   double epsilon = 1,
                                   Idx    observationPhaseLenght = 100,
                                   Idx    nbValueIterationStep = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls = new FMDPLearner< GTEST, GTEST, ITILEARNER >(
         attributeSelectionThreshold, actionReward);
      AdaptiveRMaxPlaner* rm =
         AdaptiveRMaxPlaner::TreeInstance(ls, discountFactor, epsilon);
      IPlanningStrategy< double >* ps = rm;
      IDecisionStrategy*           ds = rm;
      return new SDYNA(
         ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }

    // ==========================================================================
    // ==========================================================================
    static SDYNA* RandomMDDInstance(double attributeSelectionThreshold = 0.99,
                                    double similarityThreshold = 0.3,
                                    double discountFactor = 0.9,
                                    double epsilon = 1,
                                    Idx    observationPhaseLenght = 100,
                                    Idx    nbValueIterationStep = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls = new FMDPLearner< GTEST, GTEST, IMDDILEARNER >(
         attributeSelectionThreshold, actionReward, similarityThreshold);
      IPlanningStrategy< double >* ps =
         StructuredPlaner< double >::spumddInstance(discountFactor, epsilon);
      IDecisionStrategy* ds = new RandomDecider();
      return new SDYNA(
         ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }

    // ==========================================================================
    // ==========================================================================
    static SDYNA* RandomTreeInstance(double attributeSelectionThreshold = 0.99,
                                     double discountFactor = 0.9,
                                     double epsilon = 1,
                                     Idx    observationPhaseLenght = 100,
                                     Idx    nbValueIterationStep = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls = new FMDPLearner< CHI2TEST, CHI2TEST, ITILEARNER >(
         attributeSelectionThreshold, actionReward);
      IPlanningStrategy< double >* ps =
         StructuredPlaner< double >::sviInstance(discountFactor, epsilon);
      IDecisionStrategy* ds = new RandomDecider();
      return new SDYNA(
         ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }



    // ###################################################################
    // ###################################################################

    // ==========================================================================
    // ==========================================================================
    private:
    SDYNA(ILearningStrategy*           learner,
          IPlanningStrategy< double >* planer,
          IDecisionStrategy*           decider,
          Idx                          observationPhaseLenght,
          Idx                          nbValueIterationStep,
          bool                         actionReward,
          bool                         verbose = true);

    // ==========================================================================
    // ==========================================================================
    public:
    ~SDYNA();



    // ###################################################################
    // ###################################################################
    public:
    // ==========================================================================
    // ==========================================================================
    void addAction(const Idx actionId, const std::string& actionName) {
      _fmdp->addAction(actionId, actionName);
    }

    // ==========================================================================
    // ==========================================================================
    void addVariable(const DiscreteVariable* var) { _fmdp->addVariable(var); }



    // ###################################################################
    // ###################################################################
    public:
    // ==========================================================================
    // ==========================================================================
    void initialize();

    // ==========================================================================
    // ==========================================================================
    void initialize(const Instantiation& initialState);



    // ###################################################################
    // ###################################################################
    public:
    // ==========================================================================
    // ==========================================================================
    void setCurrentState(const Instantiation& currentState) {
      _lastState = currentState;
    }

    // ==========================================================================
    // ==========================================================================
    Idx takeAction(const Instantiation& curState);

    // ==========================================================================
    // ==========================================================================
    Idx takeAction();

    // ==========================================================================
    // ==========================================================================
    void feedback(const Instantiation& originalState,
                  const Instantiation& reachedState,
                  Idx                  performedAction,
                  double               obtainedReward);

    // ==========================================================================
    // ==========================================================================
    void feedback(const Instantiation& reachedState, double obtainedReward);

    // ==========================================================================
    // ==========================================================================
    void makePlanning(Idx nbStep);



    public:
    // ==========================================================================
    // ==========================================================================
    std::string toString();

    std::string optimalPolicy2String() { return __planer->optimalPolicy2String(); }


    // ###################################################################
    // ###################################################################
    public:
    // ==========================================================================
    // ==========================================================================
    Size learnerSize() { return __learner->size(); }

    // ==========================================================================
    // ==========================================================================
    Size modelSize() { return _fmdp->size(); }

    // ==========================================================================
    // ==========================================================================
    Size valueFunctionSize() { return __planer->vFunctionSize(); }

    // ==========================================================================
    // ==========================================================================
    Size optimalPolicySize() { return __planer->optimalPolicySize(); }



    protected:
    FMDP< double >* _fmdp;

    Instantiation _lastState;

    private:
    ILearningStrategy* __learner;

    IPlanningStrategy< double >* __planer;

    IDecisionStrategy* __decider;


    Idx __observationPhaseLenght;

    Idx __nbObservation;

    Idx __nbValueIterationStep;

    Idx __lastAction;

    Set< Observation* > __bin;

    bool __actionReward;

    bool _verbose;
  };


} /* namespace gum */


#endif   // GUM_SDYNA_H