% Figure 4.7 page 141 of text 
% Instead of V bar (approximation), V from previous iterations are stored
% and used in place of V bar.

% The algorithm avoids looping over all states.
% Stopping criteria is the number of iterations, which is fixed to an arbitrarily large number.
% This is implemented in infinite horizon. Therefore there is no time t
% Alpha decay is used for step 2b.

% use of trasition probaility is avoided
% Post decision state variable is introduced
% v is calculated around post decision state variable
clc
clf
clear
v=0;
Vmin=0;

%State 0,1,2,3 is 1,2,3,4 for Matlab
C=[	0	4000	6000
	1000	4000	6000
	3000	4000	6000
	10000	10000	6000
];

% post desion state PDS. This is a (pre-decision state on left and action
% on top) matrix, the contents of which are PDS.
PDS= [1 2 1 
      2 2 1
      3 2 1
      4 2 1];

Aph=0.95;
alph=0.7;

%***************************************
%Step 0
V=zeros(4,1);
VV(1,:)=V(:,1)';
Policy=zeros(1,4);
ite=1;
S=1; % initial state step 0c
xSx= 1; % initial Post decision state
%***************************************
%Step 1 and 2

% Step 2a  

for a =1 :1:3
    Sx= PDS(S,a);
tempv(1,a)=C(S,a)+(Aph*V(Sx,1));
end
v=min(tempv(1,:));

% Identify the action that minimized v in the previous step
for a =1 :1:3
if v-tempv(1,a)<0.001 & v-tempv(1,a)>-0.001
Act=a;
end
end

for state=1:1:4
if state==S
Policy(1,state)=Act;
end
end

%Step 2b


V(xSx,1)=(1-alph)*V(xSx,1)+alph*v
xSx=PDS(S,Act); % Find the post decision state
% alpha decay
ee=50000000*7;
    uz=ite^2/(ee+ite);
    alph=alph/(1+uz);

%Step 2c
%Finding the next predecision state (This is a random process)
f=rand(1);
if f<0.25
    Snext=1;
elseif f>=0.25 & f<0.5
    Snext =2;
elseif f>=0.5 & f<0.75
    Snext =3;
elseif f>=0.75
    Snext=4;

end
VV(2,:)=V(:,1)';
%***********************************************************
%Step 3
n=3;
while ite<1500
ite=ite+1
S=Snext;

%Step 1 and 2

% Step 2a is implemented with the assumption that all outcomes are
% possible. Therefore summation in step 2b is over all possible states as
% given in figure 4.1 page 97.

%Step 2b
for a =1 :1:3
    Sx= PDS(S,a);
tempv(1,a)=C(S,a)+(Aph*V(Sx,1));
end
v=min(tempv(1,:));

for a =1 :1:3
if v-tempv(1,a)<0.001 & v-tempv(1,a)>-0.001
Act=a;
end
end

for state=1:1:4
if state==S
Policy(1,state)=Act;
end
end



%Step 2b


V(xSx,1)=(1-alph)*V(xSx,1)+alph*v
xSx=PDS(S,Act); % Find the post decision state
% alpha decay
ee=50000000*7;
    uz=ite^2/(ee+ite);
    alph=alph/(1+uz);
VV(n,:)=V(:,1)';
n=n+1;
%Step 2c

%Finding the predecision state (This is a random process)
f=rand(1);
if f<0.25
    Snext=1;
elseif f>=0.25 & f<0.5
    Snext =2;
elseif f>=0.5 & f<0.75
    Snext =3;
elseif f>=0.75
    Snext=4;

end
end
%******************************************
Policy

plot(VV)
